yaml datasources

Travigo · Nov 5, 2024 · 83a8b88 · 83a8b88
1 parent e02f675
commit 83a8b88
Show file tree

Hide file tree

Showing 14 changed files with 563 additions and 709 deletions.
diff --git a/data/datasources/de-gtfs.yaml b/data/datasources/de-gtfs.yaml
@@ -0,0 +1,21 @@
+identifier: de-gtfs
+region: de
+provider:
+  name: GTFS.de
+  website: "https://gtfs.de"
+datasets:
+- identifier: gtfs-schedule
+  format: gtfs-schedule
+  source: "https://download.gtfs.de/germany/free/latest.zip"
+  supportedobjects:
+    operators: true
+    stops:     true
+    services:  true
+    journeys:  true
+- identifier: gtfs-realtime
+  format: gtfs-realtime
+  source: "https://realtime.gtfs.de/realtime-free.pb"
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: de-gtfs-gtfs-schedule
+  importdestination: realtime-queue
diff --git a/data/datasources/fr-ilevia-lille.yaml b/data/datasources/fr-ilevia-lille.yaml
@@ -0,0 +1,21 @@
+identifier: fr-ilevia-lille
+region: fr
+provider:
+  name: Ilévia
+  website: "http://www.ilevia.fr"
+datasets:
+- identifier: gtfs-schedule
+  format: gtfs-schedule
+  source: "https://media.ilevia.fr/opendata/gtfs.zip"
+  supportedobjects:
+    operators: true
+    stops:     true
+    services:  true
+    journeys:  true
+- identifier: gtfs-realtime
+  format: gtfs-realtime
+  source: "https://proxy.transport.data.gouv.fr/resource/ilevia-lille-gtfs-rt"
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: fr-ilevia-lille-gtfs-schedule
+  importdestination: realtime-queue
diff --git a/data/datasources/gb-dft.yaml b/data/datasources/gb-dft.yaml
@@ -0,0 +1,71 @@
+identifier: gb-dft
+region: gb
+provider:
+  name: Department for Transport
+  website: "https://www.gov.uk/government/organisations/department-for-transport"
+datasets:
+- identifier: naptan
+  format: gb-naptan
+  source: "https://naptan.api.dft.gov.uk/v1/access-nodes?dataFormat=xml"
+  supportedobjects:
+    stops:      true
+    stopgroups: true
+- identifier: bods-gtfs-schedule
+  format: gtfs-schedule
+  source: "https://data.bus-data.dft.gov.uk/timetable/download/gtfs-file/all/"
+  supportedobjects:
+    services: true
+    journeys: true
+  ignoreobjects:
+    services:
+      byoperator:
+      - gb-noc-NATX
+    journeys:
+      byoperator:
+      - gb-noc-NATX
+- identifier: bods-transxchange-coach
+  format: gb-transxchange
+  source: "https://coach.bus-data.dft.gov.uk/TxC-2.4.zip"
+  unpackbundle: zip
+  supportedobjects:
+    services: true
+    journeys: true
+  ignoreobjects:
+    services:
+      byoperator:
+      - gb-noc-APBX
+      - gb-noc-BHAT
+      - gb-noc-FLIX
+      - gb-noc-MEGA
+      - gb-noc-SCLK
+      - gb-noc-ULSL
+    journeys:
+      byoperator:
+      - gb-noc-APBX
+      - gb-noc-BHAT
+      - gb-noc-FLIX
+      - gb-noc-MEGA
+      - gb-noc-SCLK
+      - gb-noc-ULSL
+- identifier: bods-sirivm-all
+  format: eu-siri-vm
+  source: "https://data.bus-data.dft.gov.uk/avl/download/bulk_archive"
+  sourceauthentication:
+    query:
+      api_key: TRAVIGO_BODS_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: gb-dft-bods-gtfs-schedule
+  unpackbundle: zip
+  importdestination: realtime-queue
+- identifier: bods-gtfs-realtime
+  format: gtfs-realtime
+  source: "https://data.bus-data.dft.gov.uk/avl/download/gtfsrt"
+  sourceauthentication:
+    query:
+      api_key: TRAVIGO_BODS_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: gb-dft-bods-gtfs-schedule
+  unpackbundle: zip
+  importdestination: realtime-queue
diff --git a/data/datasources/gb-nationalrail.yaml b/data/datasources/gb-nationalrail.yaml
@@ -0,0 +1,22 @@
+identifier: gb-nationalrail
+region: gb
+provider:
+  name: National Rail
+  website: "https://nationalrail.co.uk"
+datasets:
+- identifier: toc
+  format: gb-nationalrailtoc
+  source: "https://opendata.nationalrail.co.uk/api/staticfeeds/4.0/tocs"
+  sourceauthentication:
+    custom: "gb-nationalrail-login"
+  supportedobjects:
+    operators: true
+    services: true
+- identifier: timetable
+  format: gb-cif
+  source: "https://opendata.nationalrail.co.uk/api/staticfeeds/3.0/timetable"
+  sourceauthentication:
+    custom: "gb-nationalrail-login"
+  supportedobjects:
+    journeys: true
+    services: true
diff --git a/data/datasources/gb-networkrail.yaml b/data/datasources/gb-networkrail.yaml
@@ -0,0 +1,16 @@
+identifier: gb-networkrail
+region: gb
+provider:
+  name: Network Rail
+  website: "https://networkrail.co.uk"
+datasets:
+- identifier: corpus
+  format: gb-networkrailcorpus
+  source: "https://publicdatafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS"
+  sourceauthentication:
+    basic:
+      username: TRAVIGO_NETWORKRAIL_USERNAME
+      password: TRAVIGO_NETWORKRAIL_PASSWORD
+  supportedobjects:
+    stops: true
+  unpackbundle: gz
diff --git a/data/datasources/gb-traveline.yaml b/data/datasources/gb-traveline.yaml
@@ -0,0 +1,12 @@
+identifier: gb-traveline
+region: gb
+provider:
+  name: Traveline
+  website: "https://www.travelinedata.org.uk/"
+datasets:
+- identifier: noc
+  format: gb-travelinenoc
+  source: "https://www.travelinedata.org.uk/noc/api/1.0/nocrecords.xml"
+  supportedobjects:
+    operators:      true
+    operatorgroups: true
diff --git a/data/datasources/ie-tfi.yaml b/data/datasources/ie-tfi.yaml
@@ -0,0 +1,24 @@
+identifier: ie-tfi
+region: ie
+provider:
+  name: Transport for Ireland
+  website: "https://www.transportforireland.ie"
+datasets:
+- identifier: gtfs-schedule
+  format: gtfs-schedule
+  source: "https://www.transportforireland.ie/transitData/Data/GTFS_Realtime.zip"
+  supportedobjects:
+    operators: true
+    stops:     true
+    services:  true
+    journeys:  true
+- identifier: gtfs-realtime
+  format: gtfs-realtime
+  source: "https://api.nationaltransport.ie/gtfsr/v2/gtfsr"
+  sourceauthentication:
+    header:
+      x-api-key: TRAVIGO_IE_NATIONALTRANSPORT_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: ie-tfi-gtfs-schedule
+  importdestination: realtime-queue
diff --git a/data/datasources/se-trafiklab.yaml b/data/datasources/se-trafiklab.yaml
@@ -0,0 +1,107 @@
+identifier: se-trafiklab
+region: se
+provider:
+  name: Trafiklab
+  website: "https://trafiklab.se"
+datasets:
+- identifier: gtfs-schedule
+  format: gtfs-schedule
+  source: "https://opendata.samtrafiken.se/gtfs-sweden/sweden.zip"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_STATIC_API_KEY
+  supportedobjects:
+    operators: true
+    stops:     true
+    services:  true
+    journeys:  true
+- identifier: gtfs-realtime-sl-trip
+  format: gtfs-realtime
+  source: "https://opendata.samtrafiken.se/gtfs-rt-sweden/sl/TripUpdatesSweden.pb"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_REALTIME_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: se-trafiklab-gtfs-schedule
+  importdestination: realtime-queue
+- identifier: gtfs-realtime-ul-trip
+  format: gtfs-realtime
+  source: "https://opendata.samtrafiken.se/gtfs-rt-sweden/ul/TripUpdatesSweden.pb"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_REALTIME_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: se-trafiklab-gtfs-schedule
+  importdestination: realtime-queue
+- identifier: gtfs-realtime-otraf-trip
+  format: gtfs-realtime
+  source: "https://opendata.samtrafiken.se/gtfs-rt-sweden/otraf/TripUpdatesSweden.pb"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_REALTIME_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: se-trafiklab-gtfs-schedule
+  importdestination: realtime-queue
+- identifier: gtfs-realtime-klt-trip
+  format: gtfs-realtime
+  source: "https://opendata.samtrafiken.se/gtfs-rt-sweden/klt/TripUpdatesSweden.pb"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_REALTIME_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: se-trafiklab-gtfs-schedule
+  importdestination: realtime-queue
+- identifier: gtfs-realtime-skane-trip
+  format: gtfs-realtime
+  source: "https://opendata.samtrafiken.se/gtfs-rt-sweden/skane/TripUpdatesSweden.pb"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_REALTIME_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: se-trafiklab-gtfs-schedule
+  importdestination: realtime-queue
+- identifier: gtfs-realtime-dt-trip
+  format: gtfs-realtime
+  source: "https://opendata.samtrafiken.se/gtfs-rt-sweden/dt/TripUpdatesSweden.pb"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_REALTIME_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: se-trafiklab-gtfs-schedule
+  importdestination: realtime-queue
+- identifier: gtfs-realtime-varm-trip
+  format: gtfs-realtime
+  source: "https://opendata.samtrafiken.se/gtfs-rt-sweden/varm/TripUpdatesSweden.pb"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_REALTIME_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: se-trafiklab-gtfs-schedule
+  importdestination: realtime-queue
+- identifier: gtfs-realtime-xt-trip
+  format: gtfs-realtime
+  source: "https://opendata.samtrafiken.se/gtfs-rt-sweden/xt/TripUpdatesSweden.pb"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_REALTIME_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: se-trafiklab-gtfs-schedule
+  importdestination: realtime-queue
+- identifier: gtfs-realtime-vastmanland-trip
+  format: gtfs-realtime
+  source: "https://opendata.samtrafiken.se/gtfs-rt-sweden/vastmanland/TripUpdatesSweden.pb"
+  sourceauthentication:
+    query:
+      key: TRAVIGO_SE_TRAFIKLAB_REALTIME_API_KEY
+  supportedobjects:
+    realtimejourneys: true
+  linkeddataset: se-trafiklab-gtfs-schedule
+  importdestination: realtime-queue
diff --git a/deploy/charts/travigo-data-importer/values.yaml b/deploy/charts/travigo-data-importer/values.yaml
@@ -61,19 +61,19 @@ cronjobs:
     args: ["data-importer", "dataset", "--id", "gb-dft-bods-transxchange-coach"]
   - name: ie-gtfs-schedule
     schedule: "50 6 * * *"
-    args: ["data-importer", "dataset", "--id", "ie-gtfs-schedule"]
-    batchburst: true
-  - name: us-nyc-subway-schedule
-    schedule: "0 8 * * *"
-    args: ["data-importer", "dataset", "--id", "us-nyc-subway-schedule"]
+    args: ["data-importer", "dataset", "--id", "ie-tfi-gtfs-schedule"]
     batchburst: true
+  # - name: us-nyc-subway-schedule
+  #   schedule: "0 8 * * *"
+  #   args: ["data-importer", "dataset", "--id", "us-nyc-subway-schedule"]
+    # batchburst: true
   - name: de-gtfs-full-schedule
     schedule: "0 7 * * *"
     args: ["data-importer", "dataset", "--id", "de-gtfs-full-schedule"]
     batchburst: true
-  - name: se-gtfs-schedule
+  - name: se-trafiklab-gtfs-schedule
     schedule: "30 7 * * *"
-    args: ["data-importer", "dataset", "--id", "se-gtfs-schedule"]
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-schedule"]
     batchburst: true
   - name: fr-ilevia-lille-gtfs-schedule
     schedule: "0 8 * * *"
@@ -92,29 +92,29 @@ deployments:
   - name: gb-bods-gtfs-rt
     args: ["data-importer", "dataset", "--id", "gb-dft-bods-gtfs-realtime", "--repeat-every", "10s"]
   - name: ie-gtfs-realtime
-    args: ["data-importer", "dataset", "--id", "ie-gtfs-realtime", "--repeat-every", "90s"]
+    args: ["data-importer", "dataset", "--id", "ie-tfi-gtfs-realtime", "--repeat-every", "90s"]
   - name: fr-ilevia-lille-gtfs-rt
     args: ["data-importer", "dataset", "--id", "fr-ilevia-lille-gtfs-realtime", "--repeat-every", "90s"]
   - name: de-gtfs-full-realtime
     args: ["data-importer", "dataset", "--id", "de-gtfs-full-realtime", "--repeat-every", "90s"]
-  - name: se-gtfs-realtime-sl-trip
-    args: ["data-importer", "dataset", "--id", "se-gtfs-realtime-sl-trip", "--repeat-every", "90s"]
-  - name: se-gtfs-realtime-ul-trip
-    args: ["data-importer", "dataset", "--id", "se-gtfs-realtime-ul-trip", "--repeat-every", "90s"]
-  - name: se-gtfs-realtime-otraf-trip
-    args: ["data-importer", "dataset", "--id", "se-gtfs-realtime-otraf-trip", "--repeat-every", "90s"]
-  - name: se-gtfs-realtime-klt-trip
-    args: ["data-importer", "dataset", "--id", "se-gtfs-realtime-klt-trip", "--repeat-every", "90s"]
-  - name: se-gtfs-realtime-skane-trip
-    args: ["data-importer", "dataset", "--id", "se-gtfs-realtime-skane-trip", "--repeat-every", "90s"]
-  - name: se-gtfs-realtime-dt-trip
-    args: ["data-importer", "dataset", "--id", "se-gtfs-realtime-dt-trip", "--repeat-every", "90s"]
-  - name: se-gtfs-realtime-varm-trip
-    args: ["data-importer", "dataset", "--id", "se-gtfs-realtime-varm-trip", "--repeat-every", "90s"]
-  - name: se-gtfs-realtime-xt-trip
-    args: ["data-importer", "dataset", "--id", "se-gtfs-realtime-xt-trip", "--repeat-every", "90s"]
-  - name: se-gtfs-realtime-vastmanland-trip
-    args: ["data-importer", "dataset", "--id", "se-gtfs-realtime-vastmanland-trip", "--repeat-every", "90s"]
+  - name: se-trafiklab-gtfs-realtime-sl-trip
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-realtime-sl-trip", "--repeat-every", "90s"]
+  - name: se-trafiklab-gtfs-realtime-ul-trip
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-realtime-ul-trip", "--repeat-every", "90s"]
+  - name: se-trafiklab-gtfs-realtime-otraf-trip
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-realtime-otraf-trip", "--repeat-every", "90s"]
+  - name: se-trafiklab-gtfs-realtime-klt-trip
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-realtime-klt-trip", "--repeat-every", "90s"]
+  - name: se-trafiklab-gtfs-realtime-skane-trip
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-realtime-skane-trip", "--repeat-every", "90s"]
+  - name: se-trafiklab-gtfs-realtime-dt-trip
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-realtime-dt-trip", "--repeat-every", "90s"]
+  - name: se-trafiklab-gtfs-realtime-varm-trip
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-realtime-varm-trip", "--repeat-every", "90s"]
+  - name: se-trafiklab-gtfs-realtime-xt-trip
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-realtime-xt-trip", "--repeat-every", "90s"]
+  - name: se-trafiklab-gtfs-realtime-vastmanland-trip
+    args: ["data-importer", "dataset", "--id", "se-trafiklab-gtfs-realtime-vastmanland-trip", "--repeat-every", "90s"]
   # - name: us-nyc-subway-realtime-numbers
   #   args: ["data-importer", "dataset", "--id", "us-nyc-subway-relatime-1-2-3-4-5-6-7", "--repeat-every", "60s"]