Streamline observations download

.prospector.yml

@@ -10,13 +10,24 @@ test-warnings: true
 member-warnings: false
 
 pyroma:
-    run: true
+  run: true
 
 pep8:
-    full: true
+  full: true
 
 pep257:
-    # disable rules that are allowed by the numpy convention
-    # see https://github.com/PyCQA/pydocstyle/blob/master/src/pydocstyle/violations.py
-    # and http://pydocstyle.readthedocs.io/en/latest/error_codes.html
-    disable: ['D107', 'D203', 'D212', 'D213', 'D402', 'D413', 'D416']
+  # disable rules that are allowed by the numpy convention
+  # see https://github.com/PyCQA/pydocstyle/blob/master/src/pydocstyle/violations.py
+  # and http://pydocstyle.readthedocs.io/en/latest/error_codes.html
+  disable:
+    - D107
+    - D203
+    - D204
+    - D212
+    - D213
+    - D402
+    - D413
+    - D416
+
+pylint:
+  run: false

.pylintrc

@@ -65,7 +65,59 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=import-star-module-level,old-octal-literal,oct-method,print-statement,unpacking-in-except,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating,import-error
+disable= apply-builtin,
+    backtick,
+    basestring-builtin,
+    buffer-builtin,
+    cmp-builtin,
+    cmp-method,
+    coerce-builtin,
+    coerce-method,
+    delslice-method,
+    dict-iter-method,
+    dict-view-method,
+    duplicate-code,
+    execfile-builtin,
+    file-builtin,
+    filter-builtin-not-iterating,
+    getslice-method,
+    hex-method,
+    import-error,
+    import-star-module-level,
+    indexing-exception,
+    input-builtin,
+    intern-builtin,
+    long-builtin,
+    long-suffix,
+    map-builtin-not-iterating,
+    metaclass-assignment,
+    next-method-called,
+    no-absolute-import,
+    nonzero-method,
+    oct-method,
+    old-division,
+    old-ne-operator,
+    old-octal-literal,
+    old-raise-syntax,
+    parameter-unpacking,
+    print-statement,
+    raising-string,
+    range-builtin-not-iterating,
+    raw_input-builtin,
+    reduce-builtin,
+    reload-builtin,
+    round-builtin,
+    setslice-method,
+    standarderror-builtin,
+    suppressed-message,
+    unichr-builtin,
+    unicode-builtin,
+    unpacking-in-except,
+    unused-argument,
+    useless-suppression,
+    using-cmp-argument,
+    xrange-builtin,
+    zip-builtin-not-iterating
 
 
 [REPORTS]
@@ -320,7 +372,7 @@ ignore-imports=no
 [DESIGN]
 
 # Maximum number of arguments for function / method
-max-args=5
+max-args=7
 
 # Argument names that match this expression will be ignored. Default to name
 # with leading underscore

doc/sphinx/source/community/dataset.rst

@@ -15,13 +15,15 @@ Dataset documentation
 The documentation required for a CMORizer script is the following:
 
 - Make sure that the new dataset is added to the list of
-  :ref:`supported_datasets`
+  :ref:`supported_datasets` and to the file datasets.yml_.
 - The in code documentation should contain clear instructions on how to obtain
-  the data
+  the data.
 - A BibTeX file named ``<dataset>.bibtex`` defining the reference for the new
   dataset should be placed in the directory ``esmvaltool/references/``, see
   :ref:`adding_references` for detailed instructions.
 
+.. _datasets.yml: https://github.com/ESMValGroup/ESMValTool/blob/main/esmvaltool/cmorizers/data/datasets.yml
+
 For more general information on writing documentation, see :ref:`documentation`.
 
 .. _dataset-test:
@@ -37,7 +39,9 @@ To test a pull request for a new CMORizer script:
 
 #. Download the data following the instructions included in the script and place
    it in the ``RAWOBS`` path specified in your ``config-user.yml``
-#. Run the CMORizer script by running ``cmorize_obs -c <config-file> -o <dataset>``
+#. If available, use the downloading script by running
+   ``esmvaltool data download --config_file <config-file>  <dataset>``
+#. Run the cmorization by running ``esmvaltool data format <config-file> <dataset>``
 #. Copy the resulting data to the ``OBS`` (for CMIP5 compliant data) or ``OBS6``
    (for CMIP6 compliant data) path specified in your
    ``config-user.yml``
@@ -74,6 +78,8 @@ Dataset description
 Check that new dataset has been added to the table of observations defined in
 the ESMValTool guide user’s guide in section :ref:`inputdata`
 (generated from ``doc/sphinx/source/input.rst``).
+Check that the new dataset has also been added to the file `datasets.yml
+<https://github.com/ESMValGroup/ESMValTool/blob/main/esmvaltool/cmorizers/data/datasets.yml>`__.
 
 BibTeX info file
 ----------------
@@ -87,11 +93,24 @@ recipe_check_obs.yml
 Check that new dataset has been added to the testing recipe
 ``esmvaltool/recipes/examples/recipe_check_obs.yml``
 
+Downloader script
+-----------------
+
+If present, check that the new downloader script
+``esmvaltool/cmorizers/data/downloaders/datasets/<dataset>.py``
+meets standards.
+This includes the following items:
+
+* Code quality checks
+
+  1. Code quality
+  2. No Codacy errors reported
+
 CMORizer script
 ---------------
 
 Check that the new CMORizer script
-``esmvaltool/cmorizers/obs/cmorize_obs_<dataset>.{py,ncl}``
+``esmvaltool/cmorizers/data/formatters/datasets/<dataset>.{py,ncl}``
 meets standards.
 This includes the following items:
 
@@ -110,13 +129,21 @@ Config file
 -----------
 
 If present, check config file ``<dataset>.yml`` in
-``esmvaltool/cmorizers/obs/cmor_config/`` for correctness.
+``esmvaltool/cmorizers/data/cmor_config/`` for correctness.
 Use ``yamllint`` to check for syntax errors and common mistakes.
 
+Run downloader script
+---------------------
+
+If available, make sure the downloader script is working by running
+``esmvaltool data download --config_file <config-file> <dataset>``
+
+
 Run CMORizer
 ------------
 
-Make sure CMORizer is working by running ``cmorize_obs -c <config-file> -o <dataset>``
+Make sure CMORizer is working by running
+``esmvaltool data format --config_file <config-file> <dataset>``
 
 Check output of CMORizer
 ------------------------