Merge pull request #43 from UNSW-CEEM/add_new_table_config

add INTERMITTENT_GEN_SCADA table and DIRECTION column

.github/workflows/cicd.yml

@@ -0,0 +1,37 @@
+name: Continuous Integration and Deployment
+
+on:
+  release:
+    types: [created]
+  workflow_dispatch:
+
+jobs:
+  # Publishes to PyPi if tests are passed and release is created
+  publish:
+    if: github.event_name == 'release' && github.event.action == 'created'
+    needs: test
+    name: Upload release to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/nemosis/
+    permissions:
+      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing
+    steps:
+      # Checkout repo
+      - name: Checkout
+        uses: actions/checkout@v3
+      # Install rye with cache
+      - name: Install the latest version of rye
+        uses: eifinger/setup-rye@v4
+        with:
+          enable-cache: true
+      # Sync dependencies
+      - name: Sync dependencies
+        run: rye sync
+      # Build
+      - name: Build
+        run: rye build
+      # Publish to PyPI
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -1,31 +1,31 @@
 .idea/*
-nemosis/__pycache__/*
+src/nemosis/__pycache__/*
 venv/*
 build/*
 dist/*
-nemosis/build/*
-nemosis/dist/*
+src/nemosis/build/*
+src/nemosis/dist/*
 gui.spec
 .pkl
 cd
 nemosis.egg-info/*
-nemosis/build/gui/xref-gui.html
-nemosis/build/gui/warngui.txt
-nemosis/build/gui/out01-Tree.toc
-nemosis/build/gui/out00-Tree.toc
-nemosis/build/gui/out00-PYZ.toc
-nemosis/build/gui/out00-PYZ.pyz
-nemosis/build/gui/out00-PKG.toc
-nemosis/build/gui/out00-PKG.pkg
-nemosis/build/gui/out00-EXE.toc
-nemosis/build/gui/out00-Analysis.toc
-nemosis/build/gui/gui.exe.manifest
-nemosis/build/gui/base_library.zip
+src/nemosis/build/gui/xref-gui.html
+src/nemosis/build/gui/warngui.txt
+src/nemosis/build/gui/out01-Tree.toc
+src/nemosis/build/gui/out00-Tree.toc
+src/nemosis/build/gui/out00-PYZ.toc
+src/nemosis/build/gui/out00-PYZ.pyz
+src/nemosis/build/gui/out00-PKG.toc
+src/nemosis/build/gui/out00-PKG.pkg
+src/nemosis/build/gui/out00-EXE.toc
+src/nemosis/build/gui/out00-Analysis.toc
+src/nemosis/build/gui/gui.exe.manifest
+src/nemosis/build/gui/base_library.zip
 env/*
-nemosis/raw_aemo_data/*
+src/nemosis/raw_aemo_data/*
 NEMOSIS.exe
-nemosis/smoke_tests.py
-nemosis/check_new_bid_table_functionality.py
+src/nemosis/smoke_tests.py
+src/nemosis/check_new_bid_table_functionality.py
 *.pyc
 smoke.py
 NEMOSIS.exe

.python-version

@@ -0,0 +1 @@
+3.12.4

README.md

@@ -71,11 +71,12 @@ To learn more about each dynamic table visit the [wiki](https://github.com/UNSW-
 You can view the dynamic tables available by printing the NEMOSIS default settings.
 
 ```python
-from nemosis import defaults
+
+from src.nemosis import defaults
 
 print(defaults.dynamic_tables)
 
-#['DISPATCHLOAD', 'DUDETAILSUMMARY', 'DUDETAIL', 'DISPATCHCONSTRAINT', 'GENCONDATA', 'DISPATCH_UNIT_SCADA', 'DISPATCHPRICE', . . .
+# ['DISPATCHLOAD', 'DUDETAILSUMMARY', 'DUDETAIL', 'DISPATCHCONSTRAINT', 'GENCONDATA', 'DISPATCH_UNIT_SCADA', 'DISPATCHPRICE', . . .
 ```
 
 #### Workflows
@@ -87,10 +88,10 @@ Your workflow may determine how you use NEMOSIS. Because the GUI relies on data
 
 ##### Dynamic data compiler
 
-`dynamic_data_compiler` can be used to download and compile data from dynamic tables.  
+`dynamic_data_compiler` can be used to download and compile data from dynamic tables.
 
 ```python
-from nemosis import dynamic_data_compiler
+from src.nemosis import dynamic_data_compiler
 
 start_time = '2017/01/01 00:00:00'
 end_time = '2017/01/01 00:05:00'
@@ -111,7 +112,7 @@ A number of options are available to configure filtering (i.e. what data NEMOSIS
 To return only a subset of a particular table's columns, use the `select_columns` argument.
 
 ```python
-from nemosis import dynamic_data_compiler
+from src.nemosis import dynamic_data_compiler
 
 price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache,
                                    select_columns=['REGIONID', 'SETTLEMENTDATE', 'RRP'])
@@ -120,7 +121,8 @@ price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache,
 To see what columns a table has, you can inspect NEMOSIS' defaults.
 
 ```python
-from nemosis import defaults
+
+from src.nemosis import defaults
 
 print(defaults.table_columns['DISPATCHPRICE'])
 # ['SETTLEMENTDATE', 'REGIONID', 'INTERVENTION', 'RRP', 'RAISE6SECRRP', 'RAISE60SECRRP', 'RAISE5MINRRP', . . .
@@ -131,17 +133,20 @@ Columns can also be filtered by value. To do this, you need provide a column to
 In the example below, the table will be filtered to only return rows where `REGIONID == 'SA1'`.
 
 ```python
-from nemosis import dynamic_data_compiler
+from src.nemosis import dynamic_data_compiler
 
-price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache, filter_cols=['REGIONID'], filter_values=(['SA1'],))
+price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache, filter_cols=['REGIONID'],
+                                   filter_values=(['SA1'],))
 ```
 
 Several filters can be applied simultaneously. A common filter is to extract pricing data excluding any physical intervention dispatch runs (`INTERVENTION == 0` is the appropriate filter, see [here](https://github.com/UNSW-CEEM/NEMOSIS/wiki/Column-Summary#intervention)). Below is an example of filtering to get data for Gladstone Unit 1 and Hornsdale Wind Farm 2 excluding any physical dispatch runs:
 
 ```python
-from nemosis import dynamic_data_compiler
+from src.nemosis import dynamic_data_compiler
 
-unit_dispatch_data = dynamic_data_compiler(start_time, end_time, 'DISPATCHLOAD', raw_data_cache, filter_cols=['DUID', 'INTERVENTION'], filter_values=(['GSTONE1', 'HDWF2'], [0]))
+unit_dispatch_data = dynamic_data_compiler(start_time, end_time, 'DISPATCHLOAD', raw_data_cache,
+                                           filter_cols=['DUID', 'INTERVENTION'],
+                                           filter_values=(['GSTONE1', 'HDWF2'], [0]))
 ```
 
 ###### Caching options
@@ -173,7 +178,7 @@ build a data cache, but then process the cache using other packages or applicati
 The example below downloads parquet data into the cache.
 
 ```python
-from nemosis import cache_compiler
+from src.nemosis import cache_compiler
 
 cache_compiler(start_time, end_time, table, raw_data_cache, fformat='parquet')
 ```
@@ -188,7 +193,8 @@ true so the additional columns are added to the cache files when they are rebuil
 columns should also work with the `cache_compiler` function.
 
 ```python
-from nemosis import defaults, dynamic_data_compiler
+from src.nemosis import dynamic_data_compiler
+from src.nemosis import defaults
 
 defaults.table_columns['BIDPEROFFER_D'] += ['PASAAVAILABILITY']
 
@@ -209,7 +215,8 @@ To learn more about each static table visit the [wiki](https://github.com/UNSW-C
 You can view the static tables available by printing the tables in NEMOSIS' defaults:
 
 ```python
-from nemosis import defaults
+
+from src.nemosis import defaults
 
 print(defaults.static_tables)
 # ['ELEMENTS_FCAS_4_SECOND', 'VARIABLES_FCAS_4_SECOND', 'Generators and Scheduled Loads', 'FCAS Providers']
@@ -220,7 +227,7 @@ print(defaults.static_tables)
 The `static_table` function can be used to access these tables
 
 ```python
-from nemosis import static_table
+from src.nemosis import static_table
 
 fcas_variables = static_table('VARIABLES_FCAS_4_SECOND', raw_data_cache)
 ```
@@ -233,7 +240,7 @@ imports, as shown below. This will disable log messages unless they are at least
 
 import logging
 
-from nemosis import dynamic_data_compiler
+from src.nemosis import dynamic_data_compiler
 
 logging.getLogger("nemosis").setLevel(logging.WARNING)
 

examples/.ipynb_checkpoints/agc_following-checkpoint.py

@@ -1,7 +1,7 @@
 import pandas as pd
 import numpy as np
 from datetime import timedelta
-from nemosis import static_table, dynamic_data_compiler
+from src.nemosis import static_table, dynamic_data_compiler
 import plotly.express as px
 
 # Specify where we will be caching the raw AEMO data.

pyproject.toml

@@ -1,6 +1,35 @@
+[project]
+name = "nemosis"
+version = "3.8.0"
+description = "A tool for accessing AEMO data."
+authors = [
+    { name = "nick-gorman", email = "n.gorman305@gmail.com"},
+    { name = "prakaa", email = "abiprakash007@gmail.com" }
+]
+dependencies = [
+    "requests>=2.32.3",
+    "pyarrow>=17.0.0",
+    "feather-format>=0.4.1",
+    "pandas>=2.2.3",
+    "xlrd>=2.0.1",
+    "beautifulsoup4>=4.12.3",
+    "openpyxl>=3.1.5",
+]
+readme = "README.md"
+requires-python = ">= 3.8"
+
 [build-system]
-requires = [
-    "setuptools>=42",
-    "wheel"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.rye]
+managed = true
+dev-dependencies = [
+    "parameterized>=0.9.0"
 ]
-build-backend = "setuptools.build_meta"
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/nemosis"]

requirements-dev.lock

@@ -0,0 +1,51 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+#   generate-hashes: false
+#   universal: false
+
+-e file:.
+beautifulsoup4==4.12.3
+    # via nemosis
+certifi==2024.8.30
+    # via requests
+charset-normalizer==3.4.0
+    # via requests
+et-xmlfile==1.1.0
+    # via openpyxl
+feather-format==0.4.1
+    # via nemosis
+idna==3.10
+    # via requests
+numpy==2.1.2
+    # via pandas
+    # via pyarrow
+openpyxl==3.1.5
+    # via nemosis
+pandas==2.2.3
+    # via nemosis
+parameterized==0.9.0
+pyarrow==17.0.0
+    # via feather-format
+    # via nemosis
+python-dateutil==2.9.0.post0
+    # via pandas
+pytz==2024.2
+    # via pandas
+requests==2.32.3
+    # via nemosis
+six==1.16.0
+    # via python-dateutil
+soupsieve==2.6
+    # via beautifulsoup4
+tzdata==2024.2
+    # via pandas
+urllib3==2.2.3
+    # via requests
+xlrd==2.0.1
+    # via nemosis

requirements.lock

@@ -0,0 +1,50 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+#   generate-hashes: false
+#   universal: false
+
+-e file:.
+beautifulsoup4==4.12.3
+    # via nemosis
+certifi==2024.8.30
+    # via requests
+charset-normalizer==3.4.0
+    # via requests
+et-xmlfile==1.1.0
+    # via openpyxl
+feather-format==0.4.1
+    # via nemosis
+idna==3.10
+    # via requests
+numpy==2.1.2
+    # via pandas
+    # via pyarrow
+openpyxl==3.1.5
+    # via nemosis
+pandas==2.2.3
+    # via nemosis
+pyarrow==17.0.0
+    # via feather-format
+    # via nemosis
+python-dateutil==2.9.0.post0
+    # via pandas
+pytz==2024.2
+    # via pandas
+requests==2.32.3
+    # via nemosis
+six==1.16.0
+    # via python-dateutil
+soupsieve==2.6
+    # via beautifulsoup4
+tzdata==2024.2
+    # via pandas
+urllib3==2.2.3
+    # via requests
+xlrd==2.0.1
+    # via nemosis