From 7263b3cad0fc1e7f6eb9f6071f9c4f6666724287 Mon Sep 17 00:00:00 2001 From: Brewster Malevich Date: Tue, 28 Dec 2021 11:54:04 -0800 Subject: [PATCH 1/3] Remove cruft script template --- .../biascorrectdownscale-precipitation.yaml | 504 ++++++++++++++++++ 1 file changed, 504 insertions(+) create mode 100644 workflows/templates/biascorrectdownscale-precipitation.yaml diff --git a/workflows/templates/biascorrectdownscale-precipitation.yaml b/workflows/templates/biascorrectdownscale-precipitation.yaml new file mode 100644 index 00000000..45754902 --- /dev/null +++ b/workflows/templates/biascorrectdownscale-precipitation.yaml @@ -0,0 +1,504 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: biascorrectdownscale-precipitation + annotations: + workflows.argoproj.io/description: >- + Bias-correction and downscaling CMIP6 GCM output for precipitation. + + This includes parameters and additional processing step specific the + precipitation workflow. + workflows.argoproj.io/tags: biascorrect,downscale,cmip6,aiqpd,dc6,qplad, qdm, pr, precipitation + workflows.argoproj.io/version: '>= 3.1.0' + labels: + component: biascorrectdownscale +spec: + workflowMetadata: + labels: + component: biascorrectdownscale + entrypoint: with-jobs + arguments: + parameters: + - name: jobs + value: | + [ + { + "target": "ssp", + "variable_id": "pr", + "historical": { "activity_id": "CMIP", "experiment_id": "historical", "table_id": "day", "variable_id": "pr", "source_id": "EC-Earth3", "institution_id": "EC-Earth-Consortium", "member_id": "r1i1p1f1", "grid_label": "gr", "version": "20200310" }, + "ssp": { "activity_id": "ScenarioMIP", "experiment_id": "ssp370", "table_id": "day", "variable_id": "pr", "source_id": "EC-Earth3", "institution_id": "EC-Earth-Consortium", "member_id": "r1i1p1f1", "grid_label": "gr", "version": "20200310" } + } + ] + templates: + + + # Loop through each job listed. + - name: with-jobs + inputs: + parameters: + - name: jobs + - name: regrid-method + value: conservative + - name: correct-wetday-frequency # "true" or "false" STRING! + value: "true" + - name: qdm-kind + value: multiplicative + - name: apply-dtr-minimum-threshold + value: "false" + steps: + - - name: variable-id-switch + template: variable-id-switch + arguments: + parameters: + - name: job + value: "{{ item }}" + withParam: "{{ inputs.parameters.jobs }}" + + + # TODO: Don't need this anymore. + - name: variable-id-switch + inputs: + parameters: + - name: job + - name: variable-id + value: "{{=jsonpath(inputs.parameters.job, '$.variable_id') }}" + steps: + - - name: parameterize-simple-job + template: parameterize + arguments: + parameters: + - name: target + value: "{{=jsonpath(inputs.parameters.job, '$.target')}}" + - name: historical + value: "{{=toJson(jsonpath(inputs.parameters.job, '$.historical'))}}" + - name: ssp + value: "{{=toJson(jsonpath(inputs.parameters.job, '$.ssp'))}}" + + + # Start work for target simulation, with target-specific configs. + - name: parameterize + inputs: + parameters: + - name: target + - name: ssp + - name: historical + - name: domainfile1x1 + value: "gs://support-c23ff1a3/domain.1x1.zarr" + - name: domainfile0p25x0p25 + value: "gs://support-c23ff1a3/domain.0p25x0p25.zarr" + steps: + - - name: historical + template: biascorrectdownscale + arguments: + parameters: + - name: simulation + value: "{{ inputs.parameters.historical }}" + - name: historical + value: "{{ inputs.parameters.historical }}" + - name: first-year + value: 1950 + - name: domainfile1x1 + value: "{{ inputs.parameters.domainfile1x1 }}" + - name: domainfile0p25x0p25 + value: "{{ inputs.parameters.domainfile0p25x0p25 }}" + when: "{{inputs.parameters.target}} == historical" + - name: ssp + template: biascorrectdownscale + arguments: + parameters: + - name: simulation + value: "{{ inputs.parameters.ssp }}" + - name: historical + value: "{{ inputs.parameters.historical }}" + - name: first-year + value: 2015 + - name: domainfile1x1 + value: "{{ inputs.parameters.domainfile1x1 }}" + - name: domainfile0p25x0p25 + value: "{{ inputs.parameters.domainfile0p25x0p25 }}" + when: "{{inputs.parameters.target}} == ssp" + + + - name: biascorrectdownscale + inputs: + parameters: + - name: simulation + - name: historical + - name: regrid-method + value: conservative + - name: first-year + - name: domainfile1x1 + - name: domainfile0p25x0p25 + - name: qdm-kind + value: multiplicative + - name: correct-wetday-frequency + value: "true" + - name: apply-dtr-minimum-threshold + value: "false" + - name: reference-zarr + # Use simulation variable_id to get reference URL. + value: >- + gs://clean-b1dbca25/reanalysis/ERA-5/F320/{{=jsonpath(inputs.parameters.simulation, '$.variable_id')}}.1995-2015.F320.zarr + dag: + tasks: + - name: get-input-clean-training-url + templateRef: + name: catalog + template: get-fsspec-url-from-parameters + arguments: + parameters: + - name: experiment-id + value: "training" + - name: activity-id + value: "{{=jsonpath(inputs.parameters.historical, '$.activity_id')}}" + - name: table-id + value: "{{=jsonpath(inputs.parameters.historical, '$.table_id')}}" + - name: variable-id + value: "{{=jsonpath(inputs.parameters.historical, '$.variable_id')}}" + - name: source-id + value: "{{=jsonpath(inputs.parameters.historical, '$.source_id')}}" + - name: institution-id + value: "{{=jsonpath(inputs.parameters.historical, '$.institution_id')}}" + - name: member-id + value: "{{=jsonpath(inputs.parameters.historical, '$.member_id')}}" + - name: grid-label + value: "{{=jsonpath(inputs.parameters.historical, '$.grid_label')}}" + - name: version + value: "{{=jsonpath(inputs.parameters.historical, '$.version')}}" + - name: base-url + value: "gs://clean-b1dbca25/cmip6" + - name: get-input-clean-simulation-url + templateRef: + name: catalog + template: get-fsspec-url-from-parameters + arguments: + parameters: + - name: experiment-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.experiment_id')}}" + - name: activity-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.activity_id')}}" + - name: table-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.table_id')}}" + - name: variable-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.variable_id')}}" + - name: source-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.source_id')}}" + - name: institution-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.institution_id')}}" + - name: member-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.member_id')}}" + - name: grid-label + value: "{{=jsonpath(inputs.parameters.simulation, '$.grid_label')}}" + - name: version + value: "{{=jsonpath(inputs.parameters.simulation, '$.version')}}" + - name: base-url + value: "gs://clean-b1dbca25/cmip6" + - name: get-output-biascorrected-url + templateRef: + name: catalog + template: get-fsspec-url-from-parameters + arguments: + parameters: + - name: experiment-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.experiment_id')}}" + - name: activity-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.activity_id')}}" + - name: table-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.table_id')}}" + - name: variable-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.variable_id')}}" + - name: source-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.source_id')}}" + - name: institution-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.institution_id')}}" + - name: member-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.member_id')}}" + - name: grid-label + value: "{{=jsonpath(inputs.parameters.simulation, '$.grid_label')}}" + - name: version + value: "{{workflow.creationTimestamp.Y}}{{workflow.creationTimestamp.m}}{{workflow.creationTimestamp.d}}{{workflow.creationTimestamp.H}}{{workflow.creationTimestamp.M}}{{workflow.creationTimestamp.S}}" + - name: base-url + value: "gs://biascorrected-492e989a/stage" + - name: get-output-downscaled-url + templateRef: + name: catalog + template: get-fsspec-url-from-parameters + arguments: + parameters: + - name: experiment-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.experiment_id')}}" + - name: activity-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.activity_id')}}" + - name: table-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.table_id')}}" + - name: variable-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.variable_id')}}" + - name: source-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.source_id')}}" + - name: institution-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.institution_id')}}" + - name: member-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.member_id')}}" + - name: grid-label + value: "{{=jsonpath(inputs.parameters.simulation, '$.grid_label')}}" + - name: version + value: "{{workflow.creationTimestamp.Y}}{{workflow.creationTimestamp.m}}{{workflow.creationTimestamp.d}}{{workflow.creationTimestamp.H}}{{workflow.creationTimestamp.M}}{{workflow.creationTimestamp.S}}" + - name: base-url + value: "gs://downscaled-288ec5ac/stage" + - name: get-biascorrected-last-year + template: get-biascorrected-last-year + depends: get-input-clean-simulation-url + arguments: + parameters: + - name: in-zarr + value: "{{ tasks.get-input-clean-simulation-url.outputs.parameters.out-url }}" + - name: biascorrect + templateRef: + name: qdm + template: main + depends: >- + get-input-clean-simulation-url + && get-input-clean-training-url + && get-biascorrected-last-year + arguments: + parameters: + - name: variable-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.variable_id')}}" + - name: reference-zarr + value: "{{ inputs.parameters.reference-zarr }}" + - name: training-zarr + value: "{{ tasks.get-input-clean-training-url.outputs.parameters.out-url }}" + - name: simulation-zarr + value: "{{ tasks.get-input-clean-simulation-url.outputs.parameters.out-url }}" + - name: out-zarr + value: "gs://scratch-170cd6ec/{{ workflow.uid }}/{{ pod.name }}/qdm-adjusted.zarr" + - name: regrid-method + value: "{{ inputs.parameters.regrid-method }}" + - name: domainfile1x1 + value: "{{ inputs.parameters.domainfile1x1 }}" + - name: correct-wetday-frequency + value: "{{ inputs.parameters.correct-wetday-frequency }}" + - name: apply-dtr-minimum-threshold + value: "{{ inputs.parameters.apply-dtr-minimum-threshold }}" + - name: qdm-kind + value: "{{ inputs.parameters.qdm-kind }}" + - name: first-year + value: "{{ inputs.parameters.first-year }}" + - name: last-year + value: "{{ tasks.get-biascorrected-last-year.outputs.parameters.last-year }}" + - name: rechunk-biascorrected + depends: biascorrect + templateRef: + name: rechunk + template: rechunk + arguments: + parameters: + - name: in-zarr + value: "{{ tasks.biascorrect.outputs.parameters.out-zarr }}" + - name: time-chunk + value: 73 + - name: lat-chunk + value: 10 + - name: lon-chunk + value: 180 + - name: cap-max-biascorrected-precip + template: cap-max-precipitation + depends: rechunk-biascorrected && get-output-biascorrected-url + arguments: + parameters: + - name: in-zarr + value: "{{ tasks.rechunk-biascorrected.outputs.parameters.out-zarr }}" + - name: out-zarr + value: "{{ tasks.get-output-biascorrected-url.outputs.parameters.out-url }}" + - name: validate-biascorrected + depends: cap-max-biascorrected-precip + templateRef: + name: qualitycontrol-check-cmip6 + template: qualitycontrol-check-cmip6 + arguments: + parameters: + - name: in-zarr + value: "{{ tasks.cap-max-biascorrected-precip.outputs.parameters.out-zarr }}" + - name: variable + value: "{{=jsonpath(inputs.parameters.simulation, '$.variable_id')}}" + - name: data + value: "bias_corrected" + - name: time + value: >- + {{=jsonpath(inputs.parameters.simulation, '$.experiment_id') == 'historical' ? 'historical' : 'future'}} + - name: create-fine-reference + templateRef: + name: qplad + template: create-fine-reference + arguments: + parameters: + - name: in-zarr + value: "{{ inputs.parameters.reference-zarr }}" + - name: regrid-method + value: "{{ inputs.parameters.regrid-method }}" + - name: domain-file + value: "{{ inputs.parameters.domainfile0p25x0p25 }}" + - name: create-coarse-reference + templateRef: + name: qplad + template: create-coarse-reference + arguments: + parameters: + - name: in-zarr + value: "{{ inputs.parameters.reference-zarr }}" + - name: regrid-method + value: "{{ inputs.parameters.regrid-method }}" + - name: domainfile0p25x0p25 + value: "{{ inputs.parameters.domainfile0p25x0p25 }}" + - name: domainfile1x1 + value: "{{ inputs.parameters.domainfile1x1 }}" + - name: preprocess-biascorrected + templateRef: + name: qplad + template: preprocess-biascorrected + depends: rechunk-biascorrected && validate-biascorrected + arguments: + parameters: + - name: in-zarr + value: "{{ tasks.rechunk-biascorrected.outputs.parameters.out-zarr }}" + - name: domain-file + value: "{{ inputs.parameters.domainfile0p25x0p25 }}" + - name: qplad + depends: >- + preprocess-biascorrected + && create-coarse-reference + && create-fine-reference + templateRef: + name: qplad + template: qplad + arguments: + parameters: + - name: variable-id + value: "{{=jsonpath(inputs.parameters.simulation, '$.variable_id')}}" + - name: simulation-zarr + value: "{{ tasks.preprocess-biascorrected.outputs.parameters.out-zarr }}" + - name: coarse-reference-zarr + value: "{{ tasks.create-coarse-reference.outputs.parameters.out-zarr }}" + - name: fine-reference-zarr + value: "{{ tasks.create-fine-reference.outputs.parameters.out-zarr }}" + - name: qdm-kind + value: "{{ inputs.parameters.qdm-kind }}" + - name: out-zarr + value: "gs://scratch-170cd6ec/{{ workflow.uid }}/{{ pod.name }}/qplad-adjusted.zarr" + - name: cap-max-downscaled-precip + template: cap-max-precipitation + depends: qplad && get-output-downscaled-url + arguments: + parameters: + - name: in-zarr + value: "{{ tasks.qplad.outputs.parameters.out-zarr }}" + - name: out-zarr + value: "{{ tasks.get-output-downscaled-url.outputs.parameters.out-url }}" + - name: validate-downscaled + depends: cap-max-downscaled-precip + templateRef: + name: qualitycontrol-check-cmip6 + template: qualitycontrol-check-cmip6 + arguments: + parameters: + - name: in-zarr + value: "{{ tasks.cap-max-downscaled-precip.outputs.parameters.out-zarr }}" + - name: variable + value: "{{=jsonpath(inputs.parameters.simulation, '$.variable_id')}}" + - name: data + value: "downscaled" + - name: time + value: >- + {{=jsonpath(inputs.parameters.simulation, '$.experiment_id') == 'historical' ? 'historical' : 'future'}} + + + # This template helps to dynamically find the last year for bias-corrected + # output based on the years available in the input simulation. + # Different experiments have different end dates and we don't want to + # make up data where it didn't already exist. + - name: get-biascorrected-last-year + inputs: + parameters: + - name: in-zarr + - name: time-variable-name + value: "time" + outputs: + parameters: + - name: last-year + valueFrom: + path: "/tmp/lastyear.txt" + script: + image: us-central1-docker.pkg.dev/downscalecmip6/private/dodola:0.10.0 + command: [ python ] + source: | + import dodola.repository + + input_zarr = "{{ inputs.parameters.in-zarr }}" + time_variable = "{{ inputs.parameters.time-variable-name }}" + + print(f"Reading {input_zarr}") + ds = dodola.repository.read(input_zarr) + + # Getting the full date is useful for debugging. + max_date = ds[time_variable].max().item() + print(f"Found {max_date=}") + + max_year = max_date.year + out_lastyear = 2014 # For historical simulations. + # For ssp* experiment simulations: + if max_year == 2099: + out_lastyear = 2099 + elif max_year >= 2100: + out_lastyear = 2100 + + print(f"Using {out_lastyear=}") + with open("/tmp/lastyear.txt", mode="w") as fl: + fl.write(str(out_lastyear)) + resources: + requests: + memory: 1Gi + cpu: "100m" + limits: + memory: 2Gi + cpu: "1000m" + activeDeadlineSeconds: 900 + retryStrategy: + limit: 4 + retryPolicy: "Always" + backoff: + duration: 30s + factor: 2 + + + - name: cap-max-precipitation + inputs: + parameters: + - name: in-zarr + - name: out-zarr + value: "gs://scratch-170cd6ec/{{ workflow.uid }}/{{ pod.name }}/capped-precip.zarr" + outputs: + parameters: + - name: out-zarr + value: "{{ inputs.parameters.out-zarr }}" + container: + image: us-central1-docker.pkg.dev/downscalecmip6/private/dodola:0.15.0 + command: [ dodola ] + args: + - "adjust-maximum-precipitation" + - "{{ inputs.parameters.in-zarr }}" + - "--out={{ inputs.parameters.out-zarr }}" + - "--threshold=3000" + resources: + requests: + memory: 2Gi + cpu: "100m" + limits: + memory: 2Gi + cpu: "1000m" + activeDeadlineSeconds: 14400 + retryStrategy: + limit: 2 + retryPolicy: "Always" + backoff: + duration: 30s + factor: 2 From 614eee43e4023903219d2d3de6943f00d8ac137d Mon Sep 17 00:00:00 2001 From: Brewster Malevich Date: Tue, 28 Dec 2021 11:55:00 -0800 Subject: [PATCH 2/3] Include new biascorrectdownscale-precipitation template to be deployed --- workflows/templates/kustomization.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/workflows/templates/kustomization.yaml b/workflows/templates/kustomization.yaml index 9d644012..1f54bdfe 100644 --- a/workflows/templates/kustomization.yaml +++ b/workflows/templates/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - qplad.yaml - biascorrectdownscale.yaml + - biascorrectdownscale-precipitation.yaml - catalog.yaml - clean-cmip6.yaml - clean-era5.yaml From 5c2ed329f772f16ac3661aafe9ed7b1a253e2db7 Mon Sep 17 00:00:00 2001 From: Brewster Malevich Date: Tue, 28 Dec 2021 11:55:28 -0800 Subject: [PATCH 3/3] Switch e2e-pr-jobs to use precipitation-specific biascorr + downscale template --- workflows/templates/e2e-pr-jobs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflows/templates/e2e-pr-jobs.yaml b/workflows/templates/e2e-pr-jobs.yaml index 5b48fbcc..441ec34c 100644 --- a/workflows/templates/e2e-pr-jobs.yaml +++ b/workflows/templates/e2e-pr-jobs.yaml @@ -49,7 +49,7 @@ spec: value: "{{ inputs.parameters.jobs }}" - - name: biascorrectdownscale templateRef: - name: biascorrectdownscale + name: biascorrectdownscale-precipitation template: with-jobs arguments: parameters: