Merge remote-tracking branch 'origin/main' into add-config-obj

.github/workflows/pr-ci.yml

@@ -89,47 +89,3 @@ jobs:
         fail_ci_if_error: true
         directory: "./reports/coverage/"
         verbose: true
-
-  pre-release-version:
-    runs-on: ubuntu-latest
-    if: startsWith(github.event.pull_request.title, 'Release v')
-    outputs:
-      version: ${{ steps.get-version.outputs.version }}
-    steps:
-      - id: get-version
-        run: |
-          fullname="${{github.event.pull_request.title}}"
-          echo "version=${fullname#"Release "}" >> $GITHUB_OUTPUT
-
-  pip-test:
-    runs-on: ubuntu-latest
-    if: startsWith(github.event.pull_request.title, 'Release v')
-    steps:
-      - uses: actions/checkout@v4
-      - uses: mamba-org/setup-micromamba@v1
-        with:
-          micromamba-version: '1.5.10-0'
-          environment-name: pipbuild
-          create-args: >-
-            python=3.11
-            pip
-            python-build
-          post-cleanup: all
-          cache-environment: true
-      - name: Build package
-        run: python -m build
-      - name: Test package install
-        run: |
-          pip install dist/calliope-*.tar.gz
-          calliope --help
-
-  pre-release:
-    needs: [test, pre-release-version, pip-test]
-    if: needs.pip-test.result == 'success' && needs.test.result == 'success' && startsWith(github.event.pull_request.title, 'Release v')
-    uses: arup-group/actions-city-modelling-lab/.github/workflows/pip-build.yml@main
-    secrets:
-      TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
-    with:
-      package_name: calliope
-      version: ${{needs.pre-release-version.outputs.version}}
-      pip_args: ''

.github/workflows/pre-release.yml

@@ -0,0 +1,62 @@
+name: Pre-release CD
+
+on:
+  pull_request:
+    branches:
+      - "main"
+
+defaults:
+  run:
+    shell: bash -l {0}
+
+concurrency:
+  # Use github.run_id on main branch
+  # Use github.event.pull_request.number on pull requests, so it's unique per pull request
+  # Use github.ref on other branches, so it's unique per branch
+  group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  pre-release-version:
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.pull_request.title, 'Release v')
+    outputs:
+      version: ${{ steps.get-version.outputs.version }}
+    steps:
+      - id: get-version
+        run: |
+          fullname="${{github.event.pull_request.title}}"
+          echo "version=${fullname#"Release "}" >> $GITHUB_OUTPUT
+
+  pip-test:
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.pull_request.title, 'Release v')
+    steps:
+      - uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          micromamba-version: '1.5.10-0'
+          environment-name: pipbuild
+          create-args: >-
+            python=3.11
+            pip
+            python-build
+          post-cleanup: all
+          cache-environment: true
+      - name: Build package
+        run: python -m build
+      - name: Test package install
+        run: |
+          pip install dist/calliope-*.tar.gz
+          calliope --help
+
+  pre-release:
+    needs: [pre-release-version, pip-test]
+    if: needs.pip-test.result == 'success' && startsWith(github.event.pull_request.title, 'Release v')
+    uses: arup-group/actions-city-modelling-lab/.github/workflows/pip-build.yml@main
+    secrets:
+      TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }}
+    with:
+      package_name: calliope
+      version: ${{needs.pre-release-version.outputs.version}}
+      pip_args: ''

.github/workflows/release.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
 
     - name: Download built package from another workflow
-      uses: dawidd6/action-download-artifact@v3
+      uses: dawidd6/action-download-artifact@v6
       with:
         name: ${{ env.PACKAGENAME }}
         workflow: pr-ci.yml

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_language_version:
 
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit # https://beta.ruff.rs/docs/usage/#github-action
-    rev: v0.6.3
+    rev: v0.8.1
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]

CHANGELOG.md

@@ -1,7 +1,23 @@
-## 0.7.0.dev5 (Unreleased)
+## 0.7.0.dev5 (2024-12-04)
 
 ### User-facing changes
 
+|changed| `template:` can now be used anywhere within YAML definition files, not just in the `nodes`, `techs` and `data_tables` sections.
+
+|changed| "An overview of the Calliope terminology" information admonition to remove self-references and improve understandability.
+Now also includes a visual depiction of how the different defined components connect together (#699).
+
+|fixed| Area-based parameters have appropriate documented units of `area` rather than `area^2` (#701).
+
+|fixed| Technology capacity lower bound constraints so that `[cap-type]_min` (e.g., `flow_cap_min`) is not always enforced if the `purchased_units` variable is active (#643).
+
+|changed| Single data entries defined in YAML indexed parameters will not be automatically broadcast along indexed dimensions.
+To achieve the same functionality as in `<v0.7.dev4`, the user must set the new `init` configuration option `broadcast_param_data` to True (#615).
+
+|changed| Helper functions are now documented on their own page within the "Defining your own math" section of the documentation (#698).
+
+|new| `where(array, condition)` math helper function to apply a where array _inside_ an expression, to enable extending component dimensions on-the-fly, and applying filtering to different components within the expression (#604, #679).
+
 |new| Data tables can inherit options from `templates`, like `techs` and `nodes` (#676).
 
 |new| dimension renaming functionality when loading from a data source, using the `rename_dims` option (#680).
@@ -15,6 +31,14 @@
 |changed| `data_sources` -> `data_tables` and `data_sources.source` -> `data_tables.data`.
 This change has occurred to avoid confusion between data "sources" and model energy "sources" (#673).
 
+### Internal changes
+
+|changed| Model definition reading is now defined in a single place (preprocess/model_definition.py).
+
+|changed| Moved YAML reading/importing functionality out of `AttrDict`. It is now part of our `io` functionality.
+
+|fixed| Avoided gurobi 12.0 incompatibility with pyomo by setting the lower bound to v6.8.2.
+
 ## 0.7.0.dev4 (2024-09-10)
 
 ### User-facing changes

docs/advanced/solver.md

@@ -2,7 +2,7 @@
 
 ## Gurobi
 
-Refer to the [Gurobi manual](https://www.gurobi.com/documentation/), which contains a list of parameters.
+Refer to the [Gurobi manual](https://docs.gurobi.com/projects/optimizer/en/current/reference/parameters.html), which contains a list of parameters.
 Simply use the names given in the documentation (e.g. "NumericFocus" to set the numerical focus value).
 For example:
 

docs/contributing.md

@@ -39,6 +39,7 @@ To create a development environment for calliope, with all libraries required fo
 1. Create the calliope mamba environment: `mamba create -n calliope -c conda-forge --file requirements/base.txt --file requirements/dev.txt gurobi::gurobi`
 1. Activate the calliope mamba environment: `mamba activate calliope`
 1. Install the calliope package into the environment, in editable mode and ignoring dependencies (we have dealt with those when creating the mamba environment): `pip install --no-deps -e .`
+1. Install the interactive python shell (necessary when testing our documentation locally): `ipython kernel install --user --name=calliope_docs_build`
 
 All together:
 
@@ -48,6 +49,7 @@ cd calliope
 mamba create -n calliope -c conda-forge --file requirements/base.txt --file requirements/dev.txt gurobi::gurobi
 mamba activate calliope
 pip install --no-deps -e .
+ipython kernel install --user --name=calliope_docs_build
 ```
 
 If installing directly with pip, you can install these libraries using the `dev` option, i.e., `pip install -e '.[dev]'`

docs/creating/config.md

@@ -41,6 +41,10 @@ To test your model pipeline, `config.init.time_subset` is a good way to limit yo
     Various capabilities are available to adjust the temporal resolution of a model on-the-fly, both by resampling or using externally-provided clustering.
     See our [time adjustment page](../advanced/time.md) for more details.
 
+!!! info "See also"
+    The full set of available configuration options is documented in the [configuration schema][model-configuration-schema].
+    This provides you with a description of each configuration option and the default which will be used if you do not provide a value.
+
 ## Deep-dive into some key configuration options
 
 ### `config.build.backend`
@@ -84,7 +88,7 @@ In fact, you can use a set of results from using `plan` model to initialise both
 ### `config.solve.solver`
 
 Possible options for solver include `glpk`, `gurobi`, `cplex`, and `cbc`.
-The interface to these solvers is done through the Pyomo library. Any [solver compatible with Pyomo](https://pyomo.readthedocs.io/en/6.5.0/solving_pyomo_models.html#supported-solvers) should work with Calliope.
+The interface to these solvers is done through the Pyomo library. Any [solver compatible with Pyomo](https://pyomo.readthedocs.io/en/latest/reference/topical/appsi/appsi.solvers.html) should work with Calliope.
 
 For solvers with which Pyomo provides more than one way to interface, the additional `solver_io` option can be used.
 In the case of Gurobi, for example, it is usually fastest to use the direct Python interface:

docs/creating/data_tables.md

@@ -18,7 +18,6 @@ In brief it is:
 * [**drop**](#selecting-dimension-values-and-dropping-dimensions): dimensions to drop from your rows/columns, e.g., a "comment" row.
 * [**add_dims**](#adding-dimensions): dimensions to add to the table after loading it in, with the corresponding value(s) to assign to the dimension index.
 * [**rename_dims**](#renaming-dimensions-on-load): dimension names to map from those defined in the data table (e.g `time`) to those used in the Calliope model (e.g. `timesteps`).
-* [**template**](#using-a-template): Reference to a [template](templates.md) from which to inherit common configuration options.
 
 When we refer to "dimensions", we mean the sets over which data is indexed in the model: `nodes`, `techs`, `timesteps`, `carriers`, `costs`.
 In addition, when loading from file, there is the _required_ dimension `parameters`.
@@ -271,7 +270,7 @@ In this section we will show some examples of loading data and provide the equiv
         cost_storage_cap.data: 150
     ```
 
-    1. To limit repetition, we have defined [templates](templates.md) for our costs.
+    1. To limit repetition, we have defined [templates](yaml.md#reusing-definitions-through-templates) for our costs.
 
 !!! info "See also"
     Our [data table loading tutorial][loading-tabular-data] has more examples of loading tabular data into your model.

docs/creating/index.md

@@ -1,25 +1,22 @@
 # Creating a model
 
-In short, a Calliope model works like this: **supply technologies** can take a **source** from outside of the modeled system and turn it into a specific **carrier** in the system.
-The model specifies one or more **nodes** along with the technologies allowed at those nodes.
-**Transmission technologies** can move the same carrier from one node to another, while **conversion technologies** can convert one carrier into another at the same node.
-**Demand technologies** remove carriers from the system through a **sink**, while **storage technologies** can store carriers at a specific node.
-Putting all of these possibilities together allows a modeller to specify as simple or as complex a model as necessary to answer a given research question.
+A Calliope model is a collection of interconnected technologies, nodes and carriers describing a real world system of flows.
+Usually, we consider those to be _energy_ flows, and most of what you will read in this documentation concerns energy systems.
+However, it is just as applicable to other types of flows, such as water!
 
-??? info "An overview of the Calliope terminology"
-    The terminology defined here is used throughout the documentation and the model code and configuration files:
+**Carriers** are commodities whose flows we track, e.g., electricity, heat, hydrogen, water, CO<sub>2</sub>.
 
-    * **Technology**: a technology that produces, consumes, converts or transports carriers.
-    * **Node**: a site which can contain multiple technologies and which may contain other nodes for carrier balancing purposes.
-    * **Source**: a source of commodity that can (or must) be used by a technology to introduce carriers into the system.
-    * **Sink**: a commodity sink that can (or must) be used by a technology to remove carriers from the system.
-    * **Carrier**: a carrier that groups technologies together into the same network, for example `electricity` or `heat`.
+**Technologies** supply, consume, convert, store or transmit _carriers_, e.g., transmission lines/pipes, batteries, power plants, wind turbines, or home appliances.
 
-    As more generally in constrained optimisation, the following terms are also used:
+**Nodes** contain groups of _technologies_ and are usually geographic, e.g., a country, municipality or a single house.
 
-    * Parameter: a fixed coefficient that enters into model equations.
-    * Variable: a variable coefficient (decision variable) that enters into model equations.
-    * Constraint: an equality or inequality expression that constrains one or several variables.
+Flows can enter the system from **sources**, e.g., energy from the sun to power a solar panel, and can exit it into **sinks**, e.g., electricity consumed by household appliances.
+Unlike _carriers_, we do not explicitly track the type of commodity described by sources and sinks.
+
+Putting all of these possibilities together allows a modeller to create a model that is as simple or complex as necessary to answer a given research question.
+Calliope's syntax ensures these models are intuitive, and easy to understand and share.
+
+![Visual description of the Calliope terminology.](../img/description_of_system.svg)
 
 !!! example
     Refer to the [examples and tutorials section](../examples/index.md) for a more practical look at how to build a Calliope model.
@@ -35,7 +32,7 @@ We distinguish between:
 - the model **definition** (your representation of a physical system in YAML).
 
 Model configuration is everything under the top-level YAML key [`config`](config.md).
-Model definition is everything else, under the top-level YAML keys [`parameters`](parameters.md), [`techs`](techs.md), [`nodes`](nodes.md), [`templates`](templates.md), and [`data_tables`](data_tables.md).
+Model definition is everything else, under the top-level YAML keys [`parameters`](parameters.md), [`techs`](techs.md), [`nodes`](nodes.md), and [`data_tables`](data_tables.md).
 
 It is possible to define alternatives to the model configuration/definition that you can refer to when you initialise your model.
 These are defined under the top-level YAML keys [`scenarios` and `overrides`](scenarios.md).
@@ -84,5 +81,4 @@ The rest of this section discusses everything you need to know to set up a model
 - An overview of [YAML as it is used in Calliope](yaml.md) - though this comes first here, you can also safely skip it and refer back to it as a reference as questions arise when you go through the model configuration and definition examples.
 - More details on the [model configuration](config.md).
 - The key parts of the model definition, first, the [technologies](techs.md), then, the [nodes](nodes.md), the locations in space where technologies can be placed.
-- How to use [technology and node templates](templates.md) to reduce repetition in the model definition.
 - Other important features to be aware of when defining your model: defining [indexed parameters](parameters.md), i.e. parameter which are not indexed over technologies and nodes, [loading tabular data](data_tables.md), and defining [scenarios and overrides](scenarios.md).

docs/creating/parameters.md

@@ -53,3 +53,29 @@ Which will add the new dimension `my_new_dim` to your model: `model.inputs.my_ne
 
 !!! warning
     The `parameter` section should not be used for large datasets (e.g., indexing over the time dimension) as it will have a high memory overhead on loading the data.
+
+## Broadcasting data along indexed dimensions
+
+If you want to set the same data for all index items, you can set the `init` [configuration option](config.md) `broadcast_param_data` to True and then use a single value in `data`:
+
+=== "Without broadcasting"
+
+    ```yaml
+    my_indexed_param:
+      data: [1, 1, 1, 1]
+      index: [my_index_val1, my_index_val2, my_index_val3, my_index_val4]
+      dims: my_new_dim
+    ```
+
+=== "With broadcasting"
+
+    ```yaml
+    my_indexed_param:
+      data: 1  # All index items will take on this value
+      index: [my_index_val1, my_index_val2, my_index_val3, my_index_val4]
+      dims: my_new_dim
+    ```
+
+!!! warning
+    The danger of broadcasting is that you maybe update `index` as a scenario override without realising that the data will be broadcast over this new index.
+    E.g., if you start with `!#yaml {data: 1, index: monetary, dims: costs}` and update it with `!#yaml {index: [monetary, emissions]}` then the `data` value of `1` will be set for both `monetary` and `emissions` index values.

docs/creating/techs.md

@@ -15,7 +15,7 @@ This establishes the basic characteristics in the optimisation model (decision v
 ??? info "Sharing configuration with templates"
 
     To share definitions between technologies and/or nodes, you can use configuration templates (the `template` key).
-    This allows a technology/node to inherit definitions from [`template` definitions](templates.md).
+    This allows a technology/node to inherit definitions from [`template` definitions](yaml.md#reusing-definitions-through-templates).
     Note that `template` is different to setting a `base_tech`.
     Setting a base_tech does not entail any configuration options being inherited;
     `base_tech` is only used when building the optimisation problem (i.e., in the `math`).