separate the template config file from the one used for tests.

cdsobs/constants.py

@@ -17,7 +17,7 @@
 
 DATE_FORMAT = "%Y-%m-%d, %H:%M:%S"
 
-CONFIG_YML = Path(cdsobs_path, "data/cdsobs_config_template.yml")
+CONFIG_YML = Path(Path(cdsobs_path).parent, "tests", "data", "cdsobs_config_test.yml")
 
 DS_TEST_NAME = "insitu-observations-woudc-ozone-total-column-and-profiles"
 

cdsobs/data/cdsobs_config_template.yml

@@ -1,23 +1,23 @@
 ingestion_databases:
   main:
-    db_user: user
-    pwd: testpass
-    host: localhost
-    port: 25432
-    db_name: baron
+    db_user: someuser
+    pwd: somepassword
+    host: somehost
+    port: 5431
+    db_name: ingestion
 catalogue_db:
-  db_user: docker
+  db_user: someuser
   pwd: docker
   host: localhost
-  port: 5433
-  db_name: cataloguedbtest
+  port: 5432
+  db_name: catalogue-dev
 s3config:
-  access_key: minioadmin
-  secret_key: minioadmin
-  host: 127.0.0.1
-  port: 9000
-  secure: false
-  namespace: cds2-obs-dev
+  access_key: somekey
+  secret_key: some_secret_key
+  host: object-store.os-api.cci2.ecmwf.int
+  port: 443
+  secure: true
+  namespace: somenamespace
 datasets:
 - name: insitu-observations-woudc-ozone-total-column-and-profiles
   lon_tile_size: 180

tests/data/cdsobs_config_test.yml

@@ -0,0 +1,107 @@
+ingestion_databases:
+  main:
+    db_user: user
+    pwd: testpass
+    host: localhost
+    port: 25432
+    db_name: baron
+catalogue_db:
+  db_user: docker
+  pwd: docker
+  host: localhost
+  port: 5433
+  db_name: cataloguedbtest
+s3config:
+  access_key: minioadmin
+  secret_key: minioadmin
+  host: 127.0.0.1
+  port: 9000
+  secure: false
+  namespace: cds2-obs-dev
+datasets:
+- name: insitu-observations-woudc-ozone-total-column-and-profiles
+  lon_tile_size: 180
+  lat_tile_size: 90
+  time_tile_size: year
+- name: insitu-observations-igra-baseline-network
+  lon_tile_size: 45
+  lat_tile_size: 45
+  available_cdm_tables:
+  - observations_table
+  - header_table
+  - station_configuration
+  - sensor_configuration
+  - uncertainty_table
+  reader:
+    IGRA: "cdsobs.ingestion.readers.sql.read_header_and_data_tables"
+    IGRA_H: "cdsobs.ingestion.readers.sql.read_singletable_data"
+- name: insitu-observations-gruan-reference-network
+  lon_tile_size: 20
+  lat_tile_size: 20
+  available_cdm_tables:
+  - observations_table
+  - header_table
+  - station_configuration
+  - sensor_configuration
+  - uncertainty_table
+- name: insitu-observations-gnss
+  lon_tile_size: 20
+  lat_tile_size: 20
+  available_cdm_tables:
+  - observations_table
+  - header_table
+  - station_configuration
+  - sensor_configuration
+  - uncertainty_table
+- name: insitu-observations-near-surface-temperature-us-climate-reference-network
+  lon_tile_size:
+    uscrn_monthly: 90
+    uscrn_daily: 90
+    uscrn_hourly: 90
+    uscrn_subhourly:
+      2006-2010: 30
+      2010-2030: 20
+  lat_tile_size:
+    uscrn_monthly: 90
+    uscrn_daily: 90
+    uscrn_hourly: 90
+    uscrn_subhourly:
+      2006-2010: 30
+      2010-2030: 20
+  available_cdm_tables:
+  - observations_table
+  - header_table
+  - station_configuration
+  - sensor_configuration
+  - uncertainty_table
+  reader: "cdsobs.ingestion.readers.sql.read_singletable_data"
+- name: insitu-comprehensive-upper-air-observation-network
+  lon_tile_size: 180
+  lat_tile_size: 90
+  read_with_spatial_batches: true
+  available_cdm_tables:
+  - observations_table
+  - header_table
+  - station_configuration
+  - era5fb_table
+  - advanced_homogenisation
+  - advanced_uncertainty
+  - uncertainty_table
+  reader: "cdsobs.ingestion.readers.cuon.read_cuon_netcdfs"
+  reader_extra_args:
+    input_dir: "test"
+- name: insitu-observations-woudc-netcdfs
+  lon_tile_size: 180
+  lat_tile_size: 90
+  reader: "cdsobs.ingestion.readers.netcdf.read_flat_netcdfs"
+  reader_extra_args:
+    input_dir: "test"
+- name: insitu-observations-ndacc
+  lon_tile_size: 180
+  lat_tile_size: 90
+  available_cdm_tables:
+  - observations_table
+  - header_table
+  - station_configuration
+  - uncertainty_table
+  reader: "cdsobs.ingestion.readers.sql.read_header_and_data_tables"