Release 2025-01-23

.env.example

@@ -20,12 +20,11 @@ DATABASE_PORT=5432
 OSM_DOWNLOAD_URL=https://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf
 SKIP_DOWNLOAD=1 # skips the download if a file already exists
 SKIP_UNCHANGED=1 # skips processing of unchanged code
-SKIP_TAG_FILTER=0 # skips tag filtering
 SKIP_WARM_CACHE=1 # skips cache warming
 WAIT_FOR_FRESH_DATA=0 # waits till the given file is from today
 ID_FILTER='' # if not empty only the objects with these ids are processed. Ids need to be prefixed with the oms_type, see https://docs.osmcode.org/osmium/latest/osmium-getid.html
 COMPUTE_DIFFS=1 # whether to create diff tables or not
-FREEZE_DATA=0 # whether to surpress updating the tables.  Works only in combination with `COMPUTE_DIFFS=1`. `0` creates a new reference point, `1` uses the last reference point to diff against.
+FREEZE_DATA=0 # whether to surpress updating the tables. Works only in combination with `COMPUTE_DIFFS=1`. `0` creates a new reference point, `1` uses the last reference point to diff against. With `1` the current run diffs against some older run, with `0` the current run diffs against the last run. See [processing/README](./processing/README.md#reference)
 
 # Token for Synology log. Leave blank to disable logging
 SYNOLOGY_LOG_TOKEN=
@@ -56,6 +55,7 @@ NEXT_PUBLIC_DO_NOT_NAVIGATE=true # Disable the navigation to fix HMR
 # PRISMA
 DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}:${DATABASE_PORT}/${DATABASE_NAME}?schema=prisma
 GEO_DATABASE_URL=postgresql://${DATABASE_USER}:${DATABASE_PASSWORD}@${DATABASE_HOST}/${DATABASE_NAME}?pool_timeout=0
+
 # LOGIN (See Bitwarden "Atlas Secrets")
 # LOGIN auf master.apis.dev.openstreetmap.org
 # Docs https://wiki.openstreetmap.org/wiki/Sandbox_for_editing
@@ -83,6 +83,10 @@ ATLAS_API_KEY=m… # (See Bitwarden "Atlas Secrets")
 # ENV: Production
 # ATLAS_API_KEY=N… # (See Bitwarden "Atlas Secrets")
 
+# Maproulette
+# https://maproulette.org/user/profile for user `osm-verkehrswende-org`
+MAPROULETTE_API_KEY=19361|5a3f1788-e18c-4d64-a343-f00bdf8e2bd7
+
 # REMINDER:
 # Update `/app/src/env.d.ts` whenever this file is changed.
 

.github/workflows/deploy-app.yml

@@ -86,11 +86,15 @@ jobs:
           host: ${{ secrets.SSH_HOST }}
           username: ${{ secrets.SSH_USERNAME }}
           password: ${{ secrets.SSH_PASSWORD }}
+          # Note on the "Cleanup docker"
+          # There might be a race condition with the other github action as we are building app an processing in parallel
+          # The 72h grace period should make sure nothing happend during the current build but old things get cleaned up
           script: |
             cd /srv
             echo "Pulling latest 'app' image"
             docker compose pull app
             echo "Restarting 'app' container"
             docker compose up app -d
-            echo "Pruning images"
-            docker image prune -fa
+            echo "Cleanup docker (containers and images) that are older than 72h"
+            docker container prune -f --filter "until=72h"
+            docker image prune -fa --filter "until=72h"

.github/workflows/deploy-processing.yml

@@ -78,11 +78,15 @@ jobs:
           host: ${{ secrets.SSH_HOST }}
           username: ${{ secrets.SSH_USERNAME }}
           password: ${{ secrets.SSH_PASSWORD }}
+          # Note on the "Cleanup docker"
+          # There might be a race condition with the other github action as we are building app an processing in parallel
+          # The 72h grace period should make sure nothing happend during the current build but old things get cleaned up
           script: |
             cd /srv
             echo "Pulling latest 'processing' image"
             docker compose pull processing
             echo "Restarting 'processing' container"
             docker compose up processing -d
-            echo "Pruning images"
-            docker image prune -fa
+            echo "Cleanup docker (containers and images) that are older than 72h"
+            docker container prune -f --filter "until=72h"
+            docker image prune -fa --filter "until=72h"

.github/workflows/restart-services.yml

@@ -34,4 +34,3 @@ jobs:
             echo "Restarting containers: tiles, db, traefik and proxy"
             docker compose pull
             docker compose -f docker-compose.yml -f docker-compose.network.yml up tiles db traefik proxy -d
-            docker image prune -fa

.github/workflows/setup-env.yml

@@ -72,11 +72,12 @@ jobs:
               -e "s|^OSM_CLIENT_ID=.*$|OSM_CLIENT_ID='${{ secrets.OSM_CLIENT_ID }}'|" \
               -e "s|^OSM_CLIENT_SECRET=.*$|OSM_CLIENT_SECRET='${{ secrets.OSM_CLIENT_SECRET }}'|" \
               -e "s|^NEXT_PUBLIC_OSM_API_URL=.*$|NEXT_PUBLIC_OSM_API_URL='${{ inputs.NEXT_PUBLIC_OSM_API_URL }}'|" \
-              -e "s|^ATLAS_API_KEY=.*$|ATLAS_API_KEY='${{ secrets.ATLAS_API_KEY }}'|" \
               -e "s|^S3_KEY=.*$|S3_KEY='${{ secrets.S3_KEY }}'|" \
               -e "s|^S3_SECRET=.*$|S3_SECRET='${{ secrets.S3_SECRET }}'|" \
               -e "s|^S3_REGION=.*$|S3_REGION='${{ secrets.S3_REGION }}'|" \
               -e "s|^S3_BUCKET=.*$|S3_BUCKET='${{ secrets.S3_BUCKET }}'|" \
+              -e "s|^ATLAS_API_KEY=.*$|ATLAS_API_KEY='${{ secrets.ATLAS_API_KEY }}'|" \
+              -e "s|^MAPROULETTE_API_KEY=.*$|MAPROULETTE_API_KEY='${{ secrets.MAPROULETTE_API_KEY }}'|" \
               -e "s|^SYNOLOGY_URL=.*$|SYNOLOGY_URL='${{ secrets.SYNOLOGY_URL }}'|" \
               -e "s|^SKIP_DOWNLOAD=.*$|SKIP_DOWNLOAD='${{ vars.SKIP_DOWNLOAD }}'|" \
               -e "s|^SKIP_UNCHANGED=.*$|SKIP_UNCHANGED='${{ vars.SKIP_UNCHANGED }}'|" \

app.Dockerfile

@@ -1,4 +1,9 @@
-FROM node:18-bullseye-slim AS base
+FROM node:18-bookworm-slim AS base
+
+RUN apt-get update && \
+    apt-get install -y gdal-bin && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
 
 WORKDIR /app
 
@@ -22,11 +27,11 @@
 RUN npx blitz@2.1.0 prisma generate
 RUN npx blitz@2.1.0 build
 
 CMD npx blitz@2.1.0 prisma migrate deploy && npx blitz@2.1.0 start -p 4000
 
 # From here on we are building the production image
 FROM base AS production
 
 RUN npm install --global pm2
 
 CMD npx blitz@2.0.10 prisma migrate deploy && exec pm2-runtime node -- ./node_modules/next/dist/bin/next start -p 4000

app/.env.production

@@ -0,0 +1,5 @@
+# Needed when running `npm run build && npm run start`
+# Ignored by .dockerignore for the build with app.Dockerfile
+NEXT_PUBLIC_APP_ORIGIN=http://127.0.0.1:4000
+DATABASE_HOST=localhost
+NEXT_PUBLIC_APP_ENV=staging

.husky/pre-push → app/.husky/pre-push

@@ -1,8 +1,9 @@
-# This runs our unit tests for `Processing`
-
-./processing/run-tests.sh
-
 # Linting, Formating and Type-Checks for `App`
 
-cd app
+cd ./app
 npm run check
+
+# This runs our unit tests for `Processing`
+
+cd ..
+./processing/run-tests.sh

app/README.md

@@ -28,23 +28,20 @@ Run `npm run dev` wich will start docker and `../processing` automatically if ne
 - ORM: [Prisma](https://blitzjs.com/docs/cli-prisma)
 - Styling: [Tailwind CSS](https://tailwindcss.com/), [Tailwind UI](https://tailwindui.com/) and [Headless UI](https://headlessui.com/)
 
-### (Tip) Testing the production bundle
+### Running the production bundle locally
 
 In the [`app/`](./app/) directory do the following:
 
 1. Make sure `npm run dev` works as expected. This will make sure all packages are patched.
-2. Create a `.env.production.local` with settings like
-   ```
-   NEXT_PUBLIC_APP_ORIGIN=http://127.0.0.1:3000
-   NEXT_PUBLIC_APP_ENV='staging' # 'staging', 'production'
-   ```
-3. Run `npm run build` and `npm run start` to test the production bundle. There is also a dockerized version of our frontend which one can run with `docker compose --profile frontend up`.
-
-### (Tip) NextJS
-
-- Favicons:
-  `icon.svg` https://nextjs.org/docs/app/api-reference/file-conventions/metadata/app-icons
-  Generator for `favicon.ico` https://realfavicongenerator.net/
+2. Double check [`.env.production`](/app/.env.production)
+3. Run `npm run build` and `npm run start` to test the production bundle.
+
+There is also a dockerized version of our frontend:
+
+```
+docker compose --profile frontend build
+docker compose --profile frontend up
+```
 
 ## Helper scripts
 

app/package.json

@@ -40,7 +40,7 @@
     "updateStaticDatasets:staging": "bun --env-file=.env --env-file=./scripts/StaticDatasets/.env.staging ./scripts/StaticDatasets/updateStaticDatasets.ts --folder-filter=",
     "updateStaticDatasets:production": "bun --env-file=.env --env-file=./scripts/StaticDatasets/.env.production ./scripts/StaticDatasets/updateStaticDatasets.ts --folder-filter=",
     "updateWfsSources": "bun --env-file=.env ./scripts/StaticDatasets/updateWfsSources.ts --folder-filter=",
-    "deleteAllStaticDatasets:dev": "bun --env-file=.env --env-file=./scripts/StaticDatasets/.env.development ./scripts/StaticDatasets/deleteAllStaticDatasets.ts",
+    "deleteAllStaticDatasets:dev": "echo \"Reminder: We cannot use this on staging, production, see https://github.com/FixMyBerlin/private-issues/issues/2283\" && bun --env-file=.env --env-file=./scripts/StaticDatasets/.env.development ./scripts/StaticDatasets/deleteAllStaticDatasets.ts",
     "updateMasks": "npm run updateMasks:createGeojson && npm run updateMasks:createPmtiles && npm run updateMasks:upload && github",
     "updateMasks:createGeojson": "bun ./scripts/RegionalMasks/createGeojson.ts && prettier \"scripts/RegionalMasks/geojson/\" --write",
     "updateMasks:createPmtiles": "bun ./scripts/RegionalMasks/createPmtiles.ts",
@@ -70,7 +70,6 @@
     "@hookform/resolvers": "3.9.0",
     "@json2csv/plainjs": "7.0.6",
     "@mapbox/mapbox-gl-draw": "^1.4.3",
-    "@popperjs/core": "^2.11.8",
     "@prisma/client": "5.21.1",
     "@tanstack/react-query": "5.59.15",
     "@turf/turf": "^7.1.0",

app/scripts/MapboxStyles/json/metadata_last_process.json

@@ -2,9 +2,9 @@
   "atlas-style-package-1": {
     "key": "atlas-style-package-1",
     "about": "Metadata on the last processing of the atlas-style-package-1 styles api response",
-    "processed_at": "24.10.2024, 14:52:54",
+    "processed_at": "26.1.2025, 06:50:00",
     "style_last_published": {
-      "published_at": "24.10.2024, 14:51:55",
+      "published_at": "11.1.2025, 07:50:59",
       "version": 8
     },
     "style_id": "cl706a84j003v14o23n2r81w7",
@@ -18,7 +18,7 @@
   "atlas-style-package-2": {
     "key": "atlas-style-package-2",
     "about": "Metadata on the last processing of the atlas-style-package-2 styles api response",
-    "processed_at": "24.10.2024, 14:52:54",
+    "processed_at": "26.1.2025, 06:50:00",
     "style_last_published": {
       "published_at": "14.8.2024, 09:44:08",
       "version": 8
@@ -34,7 +34,7 @@
   "parking": {
     "key": "parking",
     "about": "Metadata on the last processing of the parking styles api response",
-    "processed_at": "24.10.2024, 14:52:54",
+    "processed_at": "26.1.2025, 06:50:00",
     "style_last_published": {
       "published_at": "11.8.2024, 06:42:30",
       "version": 8
@@ -55,5 +55,21 @@
         }
       ]
     }
+  },
+  "atlas-style-package-3-radinfra": {
+    "key": "atlas-style-package-3-radinfra",
+    "about": "Metadata on the last processing of the atlas-style-package-3-radinfra styles api response",
+    "processed_at": "26.1.2025, 06:50:00",
+    "style_last_published": {
+      "published_at": "26.1.2025, 06:49:54",
+      "version": 8
+    },
+    "style_id": "cm5qlrsda004401sb9c3bbc6w",
+    "style_owner": "hejco",
+    "style_name": "radverkehrsatlas-style-package-3-radinfra",
+    "debug_changed_names": {
+      "about": "The folder names in Mapbox need to follow a pattern of `radinfra_[DataIdentifier]_[OptionalStyleIdentifier]`, otherwise the script will create unexpected results. During processing, we cleanup the names. If any names show up below, those need to be fixed in Mapbox to prevent errors.",
+      "changedNamesForDebugging": []
+    }
   }
 }

app/scripts/MapboxStyles/process.ts

@@ -10,7 +10,7 @@ console.log(chalk.inverse.bold('START'), __filename)
 // Configuration
 const baseMapStyle =
   'https://api.maptiler.com/maps/08357855-50d4-44e1-ac9f-ea099d9de4a5/style.json?key=ECOoUBmpqklzSCASXxcu'
-const keys = ['atlas-style-package-1', 'atlas-style-package-2', 'parking']
+const keys = ['atlas-style-package-1', 'atlas-style-package-2', 'atlas-style-package-3', 'parking']
 const apiConfigs = [
   // The order in this array specifies which sprite "wins" when sprite filenames are identical (the last entry "wins")
   {
@@ -20,20 +20,25 @@ const apiConfigs = [
     apiUrl: `https://api.mapbox.com/styles/v1/osm-verkehrswende/clev6ho1i00hd01o9bfo80n9q?fresh=true&access_token=${process.env.MAPBOX_PARKING_STYLE_ACCESS_TOKEN}`,
     mapboxGroupPrefix: 'parking_',
   },
+  {
+    key: 'atlas-style-package-3-radinfra',
+    // Style https://studio.mapbox.com/styles/hejco/cm5qlrsda004401sb9c3bbc6w/edit/#13.49/48.95568/9.13281
+    enabled: process.env.MAPBOX_STYLE_ACCESS_TOKEN,
+    apiUrl: `https://api.mapbox.com/styles/v1/hejco/cm5qlrsda004401sb9c3bbc6w?fresh=true&access_token=${process.env.MAPBOX_STYLE_ACCESS_TOKEN}`,
+    mapboxGroupPrefix: 'radinfra_',
+  },
   {
     key: 'atlas-style-package-2',
     // Style https://studio.mapbox.com/styles/hejco/clfs9mdh9007n01t6lw99gyqr/edit/#13.49/48.95568/9.13281
     enabled: process.env.MAPBOX_STYLE_ACCESS_TOKEN,
     apiUrl: `https://api.mapbox.com/styles/v1/hejco/clfs9mdh9007n01t6lw99gyqr?fresh=true&access_token=${process.env.MAPBOX_STYLE_ACCESS_TOKEN}`,
-    // Only groups with `atlas_` prefix are used
     mapboxGroupPrefix: 'atlas_',
   },
   {
     key: 'atlas-style-package-1',
     // Style https://studio.mapbox.com/styles/hejco/cl706a84j003v14o23n2r81w7/edit/#13.49/48.95568/9.13281
     enabled: process.env.MAPBOX_STYLE_ACCESS_TOKEN,
     apiUrl: `https://api.mapbox.com/styles/v1/hejco/cl706a84j003v14o23n2r81w7?fresh=true&access_token=${process.env.MAPBOX_STYLE_ACCESS_TOKEN}`,
-    // Only groups with `atlas_` prefix are used
     mapboxGroupPrefix: 'atlas_',
   },
 ].filter((c) => c.enabled)
@@ -158,12 +163,12 @@ const mergedGroupAndLayers = Object.values(groupsAndLayers)
 const styleFolder = `${groupFolder}/groups`
 fs.rmSync(styleFolder, { recursive: true, force: true })
 for (const group of mergedGroupAndLayers) {
-  const groupFile = `// Autogenerated by \`scripts/MapboxStyles/process.ts\`
-  // Do not change this file manually
+  const groupFile = `// DO NOT EDIT MANUALLY
+// This file was automatically generated by \`scripts/MapboxStyles/process.ts\`
 
-  import { MapboxStyleLayer } from '../types'
+import { MapboxStyleLayer } from '../types'
 
-  export const mapboxStyleGroupLayers_${group.folderName}: MapboxStyleLayer[] = ${JSON.stringify(
+export const mapboxStyleGroupLayers_${group.folderName}: MapboxStyleLayer[] = ${JSON.stringify(
     group.layers,
     null,
     2,
@@ -180,8 +185,8 @@ for (const group of mergedGroupAndLayers) {
 const _layers = mergedGroupAndLayers.map((g) => g.layers).flat()
 const _layerKeys = _layers.map((l) => Object.keys(l)).flat()
 const deduplicatedLayerKeys = Array.from(new Set(_layerKeys)).sort((a, b) => a.localeCompare(b))
-const typesFileContent = `// Autogenerated by \`scripts/MapboxStyles/process.ts\`
-// Do not change this file manually
+const typesFileContent = `// DO NOT EDIT MANUALLY
+// This file was automatically generated by \`scripts/MapboxStyles/process.ts\`
 
 // This type is used in \`mapboxStyleLayers\` and gives some visibility into what kind of data we fetch from Mapbox.
 export type MapboxStyleLayer = {${deduplicatedLayerKeys

app/scripts/MapboxStyles/util.ts

@@ -1,9 +1,9 @@
 import chalk from 'chalk'
 
-export const fetchStyle = async (key, url, folder) => {
+export const fetchStyle = async (key: string, url: string, folder: string) => {
   const fetchStyle = await fetch(url)
   if (!fetchStyle.ok) {
-    console.error('Fetch failed', fetchStyle)
+    console.error('Fetch failed', { fetchStyle, url })
     process.exit()
   }
   const data: any = await fetchStyle.json()

app/scripts/MapboxTilesets/datasets.ts

@@ -108,4 +108,16 @@ export const tilesetConfigs: Record<
     uploadUrl: 'https://studio.mapbox.com/tilesets/hejco.7h5266ql/#14/52.51622/13.37036',
     bbox: bboxPartOfNorthGermany,
   },
+  // There is no export function for the statistics table, so we cannot use this script to update the data.
+  // See `app/src/app/regionen/[regionSlug]/_mapData/mapDataSources/export/exportIdentifier.ts` for more.
+  // aggregated_lengths: {
+  //   sourceLayer: 'aggregated_lengths',
+  //   uploadUrl: 'https://studio.mapbox.com/tilesets/hejco.TODO/#14/52.51622/13.37036',
+  //   bbox: bboxPartOfNorthGermany,
+  // },
+  todos_lines: {
+    sourceLayer: 'todos_lines',
+    uploadUrl: 'https://studio.mapbox.com/tilesets/hejco.1yvm8bes/#14/52.51622/13.37036',
+    bbox: bboxPartOfNorthGermany,
+  },
 }