From 2193ab8e7fa3bd1795c00d7b7c09489a7c358192 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Wed, 23 Oct 2024 17:12:54 +1100 Subject: [PATCH] add INTERMITTENT_GEN_SCADA table and DIRECTION column --- .github/workflows/cicd.yml | 37 +++++ .gitignore | 36 ++-- .python-version | 1 + README.md | 37 +++-- .../agc_following-checkpoint.py | 2 +- nemosis/.DS_Store | Bin 8196 -> 0 bytes pyproject.toml | 37 ++++- requirements-dev.lock | 51 ++++++ requirements.lock | 50 ++++++ setup.py | 32 ---- {nemosis => src/nemosis}/__init__.py | 0 {nemosis => src/nemosis}/compiling to exe | 0 {nemosis => src/nemosis}/custom_errors.py | 0 {nemosis => src/nemosis}/custom_tables.py | 2 +- .../nemosis}/data_fetch_methods.py | 21 +-- {nemosis => src/nemosis}/date_generators.py | 2 +- {nemosis => src/nemosis}/defaults.py | 26 ++- {nemosis => src/nemosis}/downloader.py | 36 +++- {nemosis => src/nemosis}/favicon.ico | Bin {nemosis => src/nemosis}/filters.py | 11 +- {nemosis => src/nemosis}/gui.py | 14 +- {nemosis => src/nemosis}/hook-pandas.py | 0 .../nemosis}/processing_info_maps.py | 19 +-- {nemosis => src/nemosis}/query_wrappers.py | 2 +- {nemosis => src/nemosis}/rows.py | 154 +++++++++--------- {nemosis => src/nemosis}/value_parser.py | 2 +- {nemosis => src/nemosis}/write_file_names.py | 4 +- tests/test_data_fetch_methods.py | 17 +- tests/test_date_generators.py | 2 +- tests/test_errors_and_warnings.py | 6 +- tests/test_filters.py | 2 +- tests/test_format_options.py | 3 +- tests/test_performance_stats.py | 4 +- tests/test_processing_info_maps.py | 18 +- tests/test_query_wrappers.py | 3 +- 35 files changed, 417 insertions(+), 214 deletions(-) create mode 100644 .github/workflows/cicd.yml create mode 100644 .python-version delete mode 100644 nemosis/.DS_Store create mode 100644 requirements-dev.lock create mode 100644 requirements.lock delete mode 100644 setup.py rename {nemosis => src/nemosis}/__init__.py (100%) rename {nemosis => src/nemosis}/compiling to exe (100%) rename {nemosis => src/nemosis}/custom_errors.py (100%) rename {nemosis => src/nemosis}/custom_tables.py (99%) rename {nemosis => src/nemosis}/data_fetch_methods.py (98%) rename {nemosis => src/nemosis}/date_generators.py (99%) rename {nemosis => src/nemosis}/defaults.py (97%) rename {nemosis => src/nemosis}/downloader.py (89%) rename {nemosis => src/nemosis}/favicon.ico (100%) rename {nemosis => src/nemosis}/filters.py (92%) rename {nemosis => src/nemosis}/gui.py (98%) rename {nemosis => src/nemosis}/hook-pandas.py (100%) rename {nemosis => src/nemosis}/processing_info_maps.py (94%) rename {nemosis => src/nemosis}/query_wrappers.py (98%) rename {nemosis => src/nemosis}/rows.py (90%) rename {nemosis => src/nemosis}/value_parser.py (97%) rename {nemosis => src/nemosis}/write_file_names.py (95%) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml new file mode 100644 index 0000000..c405fe7 --- /dev/null +++ b/.github/workflows/cicd.yml @@ -0,0 +1,37 @@ +name: Continuous Integration and Deployment + +on: + release: + types: [created] + workflow_dispatch: + +jobs: + # Publishes to PyPi if tests are passed and release is created + publish: + if: github.event_name == 'release' && github.event.action == 'created' + needs: test + name: Upload release to PyPI + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/project/nemosis/ + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + steps: + # Checkout repo + - name: Checkout + uses: actions/checkout@v3 + # Install rye with cache + - name: Install the latest version of rye + uses: eifinger/setup-rye@v4 + with: + enable-cache: true + # Sync dependencies + - name: Sync dependencies + run: rye sync + # Build + - name: Build + run: rye build + # Publish to PyPI + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.gitignore b/.gitignore index 2d9fb03..37e1281 100644 --- a/.gitignore +++ b/.gitignore @@ -1,31 +1,31 @@ .idea/* -nemosis/__pycache__/* +src/nemosis/__pycache__/* venv/* build/* dist/* -nemosis/build/* -nemosis/dist/* +src/nemosis/build/* +src/nemosis/dist/* gui.spec .pkl cd nemosis.egg-info/* -nemosis/build/gui/xref-gui.html -nemosis/build/gui/warngui.txt -nemosis/build/gui/out01-Tree.toc -nemosis/build/gui/out00-Tree.toc -nemosis/build/gui/out00-PYZ.toc -nemosis/build/gui/out00-PYZ.pyz -nemosis/build/gui/out00-PKG.toc -nemosis/build/gui/out00-PKG.pkg -nemosis/build/gui/out00-EXE.toc -nemosis/build/gui/out00-Analysis.toc -nemosis/build/gui/gui.exe.manifest -nemosis/build/gui/base_library.zip +src/nemosis/build/gui/xref-gui.html +src/nemosis/build/gui/warngui.txt +src/nemosis/build/gui/out01-Tree.toc +src/nemosis/build/gui/out00-Tree.toc +src/nemosis/build/gui/out00-PYZ.toc +src/nemosis/build/gui/out00-PYZ.pyz +src/nemosis/build/gui/out00-PKG.toc +src/nemosis/build/gui/out00-PKG.pkg +src/nemosis/build/gui/out00-EXE.toc +src/nemosis/build/gui/out00-Analysis.toc +src/nemosis/build/gui/gui.exe.manifest +src/nemosis/build/gui/base_library.zip env/* -nemosis/raw_aemo_data/* +src/nemosis/raw_aemo_data/* NEMOSIS.exe -nemosis/smoke_tests.py -nemosis/check_new_bid_table_functionality.py +src/nemosis/smoke_tests.py +src/nemosis/check_new_bid_table_functionality.py *.pyc smoke.py NEMOSIS.exe diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..455808f --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12.4 diff --git a/README.md b/README.md index 4e9cd4d..e867db5 100644 --- a/README.md +++ b/README.md @@ -71,11 +71,12 @@ To learn more about each dynamic table visit the [wiki](https://github.com/UNSW- You can view the dynamic tables available by printing the NEMOSIS default settings. ```python -from nemosis import defaults + +from src.nemosis import defaults print(defaults.dynamic_tables) -#['DISPATCHLOAD', 'DUDETAILSUMMARY', 'DUDETAIL', 'DISPATCHCONSTRAINT', 'GENCONDATA', 'DISPATCH_UNIT_SCADA', 'DISPATCHPRICE', . . . +# ['DISPATCHLOAD', 'DUDETAILSUMMARY', 'DUDETAIL', 'DISPATCHCONSTRAINT', 'GENCONDATA', 'DISPATCH_UNIT_SCADA', 'DISPATCHPRICE', . . . ``` #### Workflows @@ -87,10 +88,10 @@ Your workflow may determine how you use NEMOSIS. Because the GUI relies on data ##### Dynamic data compiler -`dynamic_data_compiler` can be used to download and compile data from dynamic tables. +`dynamic_data_compiler` can be used to download and compile data from dynamic tables. ```python -from nemosis import dynamic_data_compiler +from src.nemosis import dynamic_data_compiler start_time = '2017/01/01 00:00:00' end_time = '2017/01/01 00:05:00' @@ -111,7 +112,7 @@ A number of options are available to configure filtering (i.e. what data NEMOSIS To return only a subset of a particular table's columns, use the `select_columns` argument. ```python -from nemosis import dynamic_data_compiler +from src.nemosis import dynamic_data_compiler price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache, select_columns=['REGIONID', 'SETTLEMENTDATE', 'RRP']) @@ -120,7 +121,8 @@ price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache, To see what columns a table has, you can inspect NEMOSIS' defaults. ```python -from nemosis import defaults + +from src.nemosis import defaults print(defaults.table_columns['DISPATCHPRICE']) # ['SETTLEMENTDATE', 'REGIONID', 'INTERVENTION', 'RRP', 'RAISE6SECRRP', 'RAISE60SECRRP', 'RAISE5MINRRP', . . . @@ -131,17 +133,20 @@ Columns can also be filtered by value. To do this, you need provide a column to In the example below, the table will be filtered to only return rows where `REGIONID == 'SA1'`. ```python -from nemosis import dynamic_data_compiler +from src.nemosis import dynamic_data_compiler -price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache, filter_cols=['REGIONID'], filter_values=(['SA1'],)) +price_data = dynamic_data_compiler(start_time, end_time, table, raw_data_cache, filter_cols=['REGIONID'], + filter_values=(['SA1'],)) ``` Several filters can be applied simultaneously. A common filter is to extract pricing data excluding any physical intervention dispatch runs (`INTERVENTION == 0` is the appropriate filter, see [here](https://github.com/UNSW-CEEM/NEMOSIS/wiki/Column-Summary#intervention)). Below is an example of filtering to get data for Gladstone Unit 1 and Hornsdale Wind Farm 2 excluding any physical dispatch runs: ```python -from nemosis import dynamic_data_compiler +from src.nemosis import dynamic_data_compiler -unit_dispatch_data = dynamic_data_compiler(start_time, end_time, 'DISPATCHLOAD', raw_data_cache, filter_cols=['DUID', 'INTERVENTION'], filter_values=(['GSTONE1', 'HDWF2'], [0])) +unit_dispatch_data = dynamic_data_compiler(start_time, end_time, 'DISPATCHLOAD', raw_data_cache, + filter_cols=['DUID', 'INTERVENTION'], + filter_values=(['GSTONE1', 'HDWF2'], [0])) ``` ###### Caching options @@ -173,7 +178,7 @@ build a data cache, but then process the cache using other packages or applicati The example below downloads parquet data into the cache. ```python -from nemosis import cache_compiler +from src.nemosis import cache_compiler cache_compiler(start_time, end_time, table, raw_data_cache, fformat='parquet') ``` @@ -188,7 +193,8 @@ true so the additional columns are added to the cache files when they are rebuil columns should also work with the `cache_compiler` function. ```python -from nemosis import defaults, dynamic_data_compiler +from src.nemosis import dynamic_data_compiler +from src.nemosis import defaults defaults.table_columns['BIDPEROFFER_D'] += ['PASAAVAILABILITY'] @@ -209,7 +215,8 @@ To learn more about each static table visit the [wiki](https://github.com/UNSW-C You can view the static tables available by printing the tables in NEMOSIS' defaults: ```python -from nemosis import defaults + +from src.nemosis import defaults print(defaults.static_tables) # ['ELEMENTS_FCAS_4_SECOND', 'VARIABLES_FCAS_4_SECOND', 'Generators and Scheduled Loads', 'FCAS Providers'] @@ -220,7 +227,7 @@ print(defaults.static_tables) The `static_table` function can be used to access these tables ```python -from nemosis import static_table +from src.nemosis import static_table fcas_variables = static_table('VARIABLES_FCAS_4_SECOND', raw_data_cache) ``` @@ -233,7 +240,7 @@ imports, as shown below. This will disable log messages unless they are at least import logging -from nemosis import dynamic_data_compiler +from src.nemosis import dynamic_data_compiler logging.getLogger("nemosis").setLevel(logging.WARNING) diff --git a/examples/.ipynb_checkpoints/agc_following-checkpoint.py b/examples/.ipynb_checkpoints/agc_following-checkpoint.py index 577f22b..1b4ed7f 100644 --- a/examples/.ipynb_checkpoints/agc_following-checkpoint.py +++ b/examples/.ipynb_checkpoints/agc_following-checkpoint.py @@ -1,7 +1,7 @@ import pandas as pd import numpy as np from datetime import timedelta -from nemosis import static_table, dynamic_data_compiler +from src.nemosis import static_table, dynamic_data_compiler import plotly.express as px # Specify where we will be caching the raw AEMO data. diff --git a/nemosis/.DS_Store b/nemosis/.DS_Store deleted file mode 100644 index 72ef96cdd4ea6aaad31721b231d8c695a0285e41..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8196 zcmeHM!EVz)5S^u^bz7B2K$U_x_=3ctpq5qw2_Z$H9C||vf&(DeY0}0bwL{`G4IwBO zegXIeeuQ7(7r65Q@MhMn)`=q;RG}hvqup7r=gqF)jJ+O*h*Ys%StnW{A_tXaZUIe3 z;qzQa%7tWv!3ubyfMW9SyF-l;Z98BYunbrRECZGS%fNYH0DCqUXUx8@&)U{9U>W!? z8Q|xGi^{T)$bpvf=s+VW0OT}=rJ#>IKrn713yBX3y* z4z$#8QW{PwnOP+nijvuZb48t0K}*|O1}p=k3~=tgAUS(LVJd&`Zr7vTay=?TgKw$_ zsJDJfLGKvVpf2r_k6NQD+ZjTAIUTCpX*H6$zLp#HpcSvJeNz{vrZ3LSI?lW^zc~M@ zREs*L*pJ(-qThVMM^F5?R!;g)qo{t;_sh?MYRAi6eH4YUAB5GWfP+dEDleZ0VI}Gm zqjp$niJg)@;W%^7oR?ef_10J2&CUL*+q-vnZPmTMvC;3(Ir*FG54U#?x^H{$`tLsq z2c(k65iRNCllBvOn%5DA4ol6j9)n>Y zG5x^!HnjwC01w3QMj4i>7~}6SLw-~rhVaW&0fJE;?Ew{wFFmY?`!%eiGRD%6AxoDN zOB@ILK!>PJu^mBXu12Ejb1JGTK)WM8(NFPT$bJo=}dnR&0 z<~juL?1c~*=?!ttoSCS3px9fn%#(R#irFzSGvK5RRK}{91?3nlToT?RdQG?CK_Ba4 zA7#!%;39?11JT5_>|$*#{B%TlBKI)Pp7o)VX^o>RWi3ZaCu~KnIEw^NqjQ9rnM!*+V5k4*)-AD476sEJ<=49 zgyJy|O!2)qCi402O^} D^_n(B diff --git a/pyproject.toml b/pyproject.toml index b5a3c46..71833de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,35 @@ +[project] +name = "nemosis" +version = "3.8.0" +description = "A tool for accessing AEMO data." +authors = [ + { name = "nick-gorman", email = "n.gorman305@gmail.com"}, + { name = "prakaa", email = "abiprakash007@gmail.com" } +] +dependencies = [ + "requests>=2.32.3", + "pyarrow>=17.0.0", + "feather-format>=0.4.1", + "pandas>=2.2.3", + "xlrd>=2.0.1", + "beautifulsoup4>=4.12.3", + "openpyxl>=3.1.5", +] +readme = "README.md" +requires-python = ">= 3.8" + [build-system] -requires = [ - "setuptools>=42", - "wheel" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.rye] +managed = true +dev-dependencies = [ + "parameterized>=0.9.0" ] -build-backend = "setuptools.build_meta" \ No newline at end of file + +[tool.hatch.metadata] +allow-direct-references = true + +[tool.hatch.build.targets.wheel] +packages = ["src/nemosis"] diff --git a/requirements-dev.lock b/requirements-dev.lock new file mode 100644 index 0000000..4d921a5 --- /dev/null +++ b/requirements-dev.lock @@ -0,0 +1,51 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false +# generate-hashes: false +# universal: false + +-e file:. +beautifulsoup4==4.12.3 + # via nemosis +certifi==2024.8.30 + # via requests +charset-normalizer==3.4.0 + # via requests +et-xmlfile==1.1.0 + # via openpyxl +feather-format==0.4.1 + # via nemosis +idna==3.10 + # via requests +numpy==2.1.2 + # via pandas + # via pyarrow +openpyxl==3.1.5 + # via nemosis +pandas==2.2.3 + # via nemosis +parameterized==0.9.0 +pyarrow==17.0.0 + # via feather-format + # via nemosis +python-dateutil==2.9.0.post0 + # via pandas +pytz==2024.2 + # via pandas +requests==2.32.3 + # via nemosis +six==1.16.0 + # via python-dateutil +soupsieve==2.6 + # via beautifulsoup4 +tzdata==2024.2 + # via pandas +urllib3==2.2.3 + # via requests +xlrd==2.0.1 + # via nemosis diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..f7d0134 --- /dev/null +++ b/requirements.lock @@ -0,0 +1,50 @@ +# generated by rye +# use `rye lock` or `rye sync` to update this lockfile +# +# last locked with the following flags: +# pre: false +# features: [] +# all-features: false +# with-sources: false +# generate-hashes: false +# universal: false + +-e file:. +beautifulsoup4==4.12.3 + # via nemosis +certifi==2024.8.30 + # via requests +charset-normalizer==3.4.0 + # via requests +et-xmlfile==1.1.0 + # via openpyxl +feather-format==0.4.1 + # via nemosis +idna==3.10 + # via requests +numpy==2.1.2 + # via pandas + # via pyarrow +openpyxl==3.1.5 + # via nemosis +pandas==2.2.3 + # via nemosis +pyarrow==17.0.0 + # via feather-format + # via nemosis +python-dateutil==2.9.0.post0 + # via pandas +pytz==2024.2 + # via pandas +requests==2.32.3 + # via nemosis +six==1.16.0 + # via python-dateutil +soupsieve==2.6 + # via beautifulsoup4 +tzdata==2024.2 + # via pandas +urllib3==2.2.3 + # via requests +xlrd==2.0.1 + # via nemosis diff --git a/setup.py b/setup.py deleted file mode 100644 index 0616d8b..0000000 --- a/setup.py +++ /dev/null @@ -1,32 +0,0 @@ -import setuptools - -with open("README.md", "r") as fh: - long_description = fh.read() - -setuptools.setup( - name="nemosis", - version="3.7.0", - author="Nicholas Gorman, Abhijith Prakash", - author_email="n.gorman305@gmail.com", - description="A tool for accessing AEMO data.", - long_description="A tool for accessing AEMO data.", - long_description_content_type="text/markdown", - url="https://github.com/UNSW-CEEM/NEMOSIS", - packages=setuptools.find_packages(), - install_requires=[ - "requests", - "joblib", - "pyarrow", - "feather-format", - "pandas", - "xlrd", - "beautifulsoup4", - "openpyxl", - "parameterized" - ], - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: GNU General Public License (GPL)", - "Operating System :: OS Independent", - ], -) diff --git a/nemosis/__init__.py b/src/nemosis/__init__.py similarity index 100% rename from nemosis/__init__.py rename to src/nemosis/__init__.py diff --git a/nemosis/compiling to exe b/src/nemosis/compiling to exe similarity index 100% rename from nemosis/compiling to exe rename to src/nemosis/compiling to exe diff --git a/nemosis/custom_errors.py b/src/nemosis/custom_errors.py similarity index 100% rename from nemosis/custom_errors.py rename to src/nemosis/custom_errors.py diff --git a/nemosis/custom_tables.py b/src/nemosis/custom_tables.py similarity index 99% rename from nemosis/custom_tables.py rename to src/nemosis/custom_tables.py index 1b6061c..c39d7df 100644 --- a/nemosis/custom_tables.py +++ b/src/nemosis/custom_tables.py @@ -2,7 +2,7 @@ from datetime import timedelta, datetime import math import numpy as np -from . import defaults, data_fetch_methods, filters +from nemosis import defaults, data_fetch_methods, filters def fcas4s_scada_match( diff --git a/nemosis/data_fetch_methods.py b/src/nemosis/data_fetch_methods.py similarity index 98% rename from nemosis/data_fetch_methods.py rename to src/nemosis/data_fetch_methods.py index a4c9238..73d1285 100644 --- a/nemosis/data_fetch_methods.py +++ b/src/nemosis/data_fetch_methods.py @@ -2,15 +2,14 @@ import os as _os import glob as _glob import pandas as _pd -import warnings from datetime import datetime as _datetime -from . import filters as _filters -from . import downloader as _downloader -from . import processing_info_maps as _processing_info_maps -from . import defaults as _defaults -from . import custom_tables as _custom_tables -from . import _infer_column_data_types -from .custom_errors import UserInputError, NoDataToReturn, DataMismatchError +from nemosis import filters as _filters +from nemosis import downloader as _downloader +from nemosis import processing_info_maps as _processing_info_maps +from nemosis import defaults as _defaults +from nemosis import custom_tables as _custom_tables +from nemosis import _infer_column_data_types +from nemosis.custom_errors import UserInputError, NoDataToReturn, DataMismatchError logger = logging.getLogger(__name__) @@ -354,7 +353,7 @@ def _get_read_function(fformat, table_type, day): func = _read_mms_csv else: func = _read_constructed_csv - elif table_type in ['DAILY_REGION_SUMMARY', "NEXT_DAY_DISPATCHLOAD"]: + elif table_type in ['DAILY_REGION_SUMMARY', "NEXT_DAY_DISPATCHLOAD", "INTERMITTENT_GEN_SCADA"]: func = _read_constructed_csv return func @@ -727,7 +726,8 @@ def _determine_columns_and_read_csv( else: type = str if ( - _defaults.table_types[table_name] in ["MMS", "BIDDING", "DAILY_REGION_SUMMARY", "NEXT_DAY_DISPATCHLOAD"] + _defaults.table_types[table_name] in ["MMS", "BIDDING", "DAILY_REGION_SUMMARY", "NEXT_DAY_DISPATCHLOAD", + "INTERMITTENT_GEN_SCADA"] and not read_all_columns ): headers = read_csv_func(csv_file, nrows=1).columns.tolist() @@ -867,5 +867,6 @@ def _static_table_wrapper_for_gui( "MARKET_PRICE_THRESHOLDS": _dynamic_data_wrapper_for_gui, "DAILY_REGION_SUMMARY": _dynamic_data_wrapper_for_gui, "NEXT_DAY_DISPATCHLOAD": _dynamic_data_wrapper_for_gui, + "INTERMITTENT_GEN_SCADA": _dynamic_data_wrapper_for_gui, "ROOFTOP_PV_ACTUAL": _dynamic_data_wrapper_for_gui } diff --git a/nemosis/date_generators.py b/src/nemosis/date_generators.py similarity index 99% rename from nemosis/date_generators.py rename to src/nemosis/date_generators.py index b020887..2c986e7 100644 --- a/nemosis/date_generators.py +++ b/src/nemosis/date_generators.py @@ -1,5 +1,5 @@ import logging -from . import defaults +from nemosis import defaults from calendar import monthrange from datetime import timedelta diff --git a/nemosis/defaults.py b/src/nemosis/defaults.py similarity index 97% rename from nemosis/defaults.py rename to src/nemosis/defaults.py index 28411a0..8d3c16f 100644 --- a/nemosis/defaults.py +++ b/src/nemosis/defaults.py @@ -8,6 +8,7 @@ "DISPATCHCONSTRAINT": "PUBLIC_DVD_DISPATCHCONSTRAINT", "GENCONDATA": "PUBLIC_DVD_GENCONDATA", "DISPATCH_UNIT_SCADA": "PUBLIC_DVD_DISPATCH_UNIT_SCADA", + "INTERMITTENT_GEN_SCADA": "PUBLIC_NEXT_DAY_INTERMITTENT_GEN_SCADA", "DISPATCHPRICE": "PUBLIC_DVD_DISPATCHPRICE", "SPDREGIONCONSTRAINT": "PUBLIC_DVD_SPDREGIONCONSTRAINT", "SPDCONNECTIONPOINTCONSTRAINT": "PUBLIC_DVD_SPDCONNECTIONPOINTCONSTRAINT", @@ -42,6 +43,7 @@ "FCAS Providers": "STATICXL", "DISPATCHLOAD": "MMS", "NEXT_DAY_DISPATCHLOAD": "NEXT_DAY_DISPATCHLOAD", + "INTERMITTENT_GEN_SCADA": "INTERMITTENT_GEN_SCADA", "DUDETAILSUMMARY": "MMS", "PARTICIPANT": "MMS", "DUDETAIL": "MMS", @@ -80,7 +82,7 @@ dynamic_tables = [ table for table, type in table_types.items() - if type in ["MMS", "BIDDING", "DAILY_REGION_SUMMARY", "NEXT_DAY_DISPATCHLOAD", "FCAS"] + if type in ["MMS", "BIDDING", "DAILY_REGION_SUMMARY", "NEXT_DAY_DISPATCHLOAD", "INTERMITTENT_GEN_SCADA", "FCAS"] ] return_tables = list(names.keys()) @@ -89,6 +91,7 @@ "FCAS Providers", "DISPATCHLOAD", "NEXT_DAY_DISPATCHLOAD", + "INTERMITTENT_GEN_SCADA", "DUDETAILSUMMARY", "PARTICIPANT", "DUDETAIL", @@ -137,7 +140,8 @@ current_data_page_urls = { "BIDDING": "Reports/Current/Bidmove_Complete/", "DAILY_REGION_SUMMARY": "/Reports/Current/Daily_Reports/", - "NEXT_DAY_DISPATCHLOAD": "/Reports/Current/NEXT_DAY_DISPATCH/" + "NEXT_DAY_DISPATCHLOAD": "/Reports/Current/NEXT_DAY_DISPATCH/", + "INTERMITTENT_GEN_SCADA": "/Reports/Current/Next_Day_Intermittent_Gen_Scada/" } fcas_4_url = "http://www.nemweb.com.au/Reports/Current/Causer_Pays/FCAS_{}{}{}{}.zip" @@ -276,6 +280,13 @@ "LOWERREGENABLEMENTMAX", "LOWERREGENABLEMENTMIN", ], + "INTERMITTENT_GEN_SCADA": [ + "RUN_DATETIME", + "DUID", + "SCADA_TYPE", + "SCADA_VALUE", + "SCADA_QUALITY" + ], "TRADINGLOAD": [ "SETTLEMENTDATE", "DUID", @@ -465,6 +476,7 @@ "BANDAVAIL10", "MAXAVAIL", "BIDTYPE", + "DIRECTION", "SETTLEMENTDATE", "ENABLEMENTMIN", "ENABLEMENTMAX", @@ -508,6 +520,7 @@ "SETTLEMENTDATE", "DUID", "BIDTYPE", + "DIRECTION", "OFFERDATE", "VERSIONNO", "PRICEBAND1", @@ -748,6 +761,7 @@ "OFFERDATE", "INTERVAL_DATETIME", "SETTLEMENTDATE", + "DIRECTION" ], "DISPATCHINTERCONNECTORRES": [ "DISPATCHINTERVAL", @@ -757,7 +771,7 @@ ], "INTERCONNECTOR": ["INTERCONNECTORID"], "DISPATCHPRICE": ["INTERVENTION", "REGIONID", "SETTLEMENTDATE"], - "BIDDAYOFFER_D": ["BIDTYPE", "DUID", "SETTLEMENTDATE"], + "BIDDAYOFFER_D": ["BIDTYPE", "DUID", "SETTLEMENTDATE", "DIRECTION"], "DISPATCHREGIONSUM": [ "DISPATCHINTERVAL", "INTERVENTION", @@ -766,6 +780,11 @@ ], "DISPATCHLOAD": ["SETTLEMENTDATE", "INTERVENTION", "DUID"], "NEXT_DAY_DISPATCHLOAD": ["SETTLEMENTDATE", "INTERVENTION", "DUID"], + "INTERMITTENT_GEN_SCADA": [ + "RUN_DATETIME", + "DUID", + "SCADA_TYPE", + ], "DISPATCH_UNIT_SCADA": ["SETTLEMENTDATE", "DUID"], "FCAS_4_SECOND": ["TIMESTAMP", "ELEMENTNUMBER", "VARIABLENUMBER"], "ELEMENTS_FCAS_4_SECOND": ["ELEMENTNUMBER"], @@ -812,6 +831,7 @@ primary_date_columns = { "DISPATCHLOAD": "SETTLEMENTDATE", "NEXT_DAY_DISPATCHLOAD": "SETTLEMENTDATE", + "INTERMITTENT_GEN_SCADA": "RUN_DATETIME", "TRADINGLOAD": "SETTLEMENTDATE", "TRADINGPRICE": "SETTLEMENTDATE", "TRADINGREGIONSUM": "SETTLEMENTDATE", diff --git a/nemosis/downloader.py b/src/nemosis/downloader.py similarity index 89% rename from nemosis/downloader.py rename to src/nemosis/downloader.py index 001b0ed..dac391c 100644 --- a/nemosis/downloader.py +++ b/src/nemosis/downloader.py @@ -5,7 +5,6 @@ import zipfile import io import pandas as pd -from urllib.parse import quote from . import defaults, custom_errors @@ -76,6 +75,16 @@ def run_next_dispatch_tables(year, month, day, chunk, index, filename_stub, down logger.warning(f"{filename_stub} not downloaded") +def run_intermittent_gen_scada(year, month, day, chunk, index, filename_stub, down_load_to): + try: + download_url = _get_current_url( + filename_stub, + defaults.current_data_page_urls["INTERMITTENT_GEN_SCADA"]) + _download_and_unpack_intermittent_gen_scada_file(download_url, down_load_to) + except Exception: + logger.warning(f"{filename_stub} not downloaded") + + def _get_current_url(filename_stub, current_page_url): sub_url = _get_matching_link( url=defaults.nem_web_domain_url + current_page_url, @@ -173,6 +182,31 @@ def _download_and_unpack_next_dispatch_load_files_complete_files( ) +def _download_and_unpack_intermittent_gen_scada_file( + download_url, down_load_to +): + r = requests.get(download_url, headers=USR_AGENT_HEADER) + zipped_file = zipfile.ZipFile(io.BytesIO(r.content)) + + file_name = zipped_file.namelist()[ + 0 + ] # Just one file so we can pull it out of the list using 0 + start_row_second_table = _find_start_row_nth_table( + zipped_file, file_name, 1 + ) + csv_file = zipped_file.open(file_name) + data = pd.read_csv( + csv_file, header=1, dtype=str + )[:-1] + data.to_csv( + os.path.join( + down_load_to, + "PUBLIC_NEXT_DAY_INTERMITTENT_GEN_SCADA_" + file_name[39:47] + ".csv", + ), + index=False, + ) + + def _find_start_row_nth_table(sub_folder_zipfile, file_name, n): row = 0 table_start_rows_found = 0 diff --git a/nemosis/favicon.ico b/src/nemosis/favicon.ico similarity index 100% rename from nemosis/favicon.ico rename to src/nemosis/favicon.ico diff --git a/nemosis/filters.py b/src/nemosis/filters.py similarity index 92% rename from nemosis/filters.py rename to src/nemosis/filters.py index 9517a2a..f4a0247 100644 --- a/nemosis/filters.py +++ b/src/nemosis/filters.py @@ -1,9 +1,8 @@ import logging -import pandas as pd from datetime import datetime, timedelta import numpy as np -from .value_parser import _parse_datetime +from nemosis.value_parser import _parse_datetime logger = logging.getLogger(__name__) @@ -33,6 +32,14 @@ def filter_on_settlementdate(data, start_time, end_time): return data +def filter_on_run_datetime(data, start_time, end_time): + data["RUN_DATETIME"] = _parse_datetime(data["RUN_DATETIME"]) + data = data[ + (data["RUN_DATETIME"] > start_time) & (data["RUN_DATETIME"] <= end_time) + ] + return data + + def filter_on_timestamp(data, start_time, end_time): try: data["TIMESTAMP"] = _parse_datetime(data["TIMESTAMP"]) diff --git a/nemosis/gui.py b/src/nemosis/gui.py similarity index 98% rename from nemosis/gui.py rename to src/nemosis/gui.py index 460dc68..2adfb42 100644 --- a/nemosis/gui.py +++ b/src/nemosis/gui.py @@ -1,6 +1,4 @@ -from nemosis import rows -from nemosis import defaults -from nemosis import data_fetch_methods +from nemosis import rows, defaults, data_fetch_methods import pandas as pd import tkinter as tk import tkinter.ttk as ttk @@ -190,8 +188,8 @@ def add_plus(self): self.row_adder = ttk.Frame(self.frame.interior) self.row_adder.grid( row=defaults.query_row_offset - + len(self.rows) * defaults.row_height - + defaults.plus_internal_row, + + len(self.rows) * defaults.row_height + + defaults.plus_internal_row, column=0, padx=defaults.standard_x_pad, sticky="w", @@ -565,8 +563,8 @@ def replace_plus(self): # Move the plus buttons to below all existing rows. self.row_adder.grid( row=defaults.query_row_offset - + (len(self.rows)) * defaults.row_height - + defaults.plus_internal_row + + (len(self.rows)) * defaults.row_height + + defaults.plus_internal_row ) self.row_adder.update() self.frame.update_scrollbar(None) @@ -578,7 +576,7 @@ def resource_path(relative_path): # PyInstaller creates a temp folder and stores path in _MEIPASS base_path = sys._MEIPASS except Exception: - base_path = os.path.abspath(".") + base_path = os.path.abspath("") return os.path.join(base_path, relative_path) diff --git a/nemosis/hook-pandas.py b/src/nemosis/hook-pandas.py similarity index 100% rename from nemosis/hook-pandas.py rename to src/nemosis/hook-pandas.py diff --git a/nemosis/processing_info_maps.py b/src/nemosis/processing_info_maps.py similarity index 94% rename from nemosis/processing_info_maps.py rename to src/nemosis/processing_info_maps.py index f058e75..768a1a3 100644 --- a/nemosis/processing_info_maps.py +++ b/src/nemosis/processing_info_maps.py @@ -1,17 +1,9 @@ -import os -from . import query_wrappers - -from nemosis import ( - filters, - downloader, - write_file_names, - date_generators, -) - +from nemosis import query_wrappers, downloader, date_generators, write_file_names, filters setup = { "DISPATCHLOAD": None, "NEXT_DAY_DISPATCHLOAD": None, + "INTERMITTENT_GEN_SCADA": None, "TRADINGLOAD": None, "TRADINGPRICE": None, "TRADINGREGIONSUM": None, @@ -50,6 +42,7 @@ search_type = { "DISPATCHLOAD": "start_to_end", "NEXT_DAY_DISPATCHLOAD": "start_to_end", + "INTERMITTENT_GEN_SCADA": "start_to_end", "TRADINGLOAD": "start_to_end", "TRADINGPRICE": "start_to_end", "TRADINGREGIONSUM": "start_to_end", @@ -88,6 +81,7 @@ date_cols = { "DISPATCHLOAD": ["SETTLEMENTDATE"], "NEXT_DAY_DISPATCHLOAD": ["SETTLEMENTDATE"], + "INTERMITTENT_GEN_SCADA": ["RUN_DATETIME"], "TRADINGLOAD": ["SETTLEMENTDATE"], "TRADINGPRICE": ["SETTLEMENTDATE"], "TRADINGREGIONSUM": ["SETTLEMENTDATE"], @@ -126,6 +120,7 @@ filter = { "DISPATCHLOAD": filters.filter_on_settlementdate, "NEXT_DAY_DISPATCHLOAD": filters.filter_on_settlementdate, + "INTERMITTENT_GEN_SCADA": filters.filter_on_run_datetime, "TRADINGLOAD": filters.filter_on_settlementdate, "TRADINGPRICE": filters.filter_on_settlementdate, "TRADINGREGIONSUM": filters.filter_on_settlementdate, @@ -164,6 +159,7 @@ finalise = { "DISPATCHLOAD": None, "NEXT_DAY_DISPATCHLOAD": None, + "INTERMITTENT_GEN_SCADA": None, "TRADINGLOAD": None, "TRADINGPRICE": None, "TRADINGREGIONSUM": None, @@ -245,6 +241,7 @@ date_gen = { "MMS": date_generators.year_and_month_gen, "NEXT_DAY_DISPATCHLOAD": date_generators.current_gen, + "INTERMITTENT_GEN_SCADA": date_generators.current_gen, "BIDDING": date_generators.bid_table_gen, "DAILY_REGION_SUMMARY": date_generators.current_gen, "FCAS": date_generators.year_month_day_index_gen, @@ -253,6 +250,7 @@ write_filename = { "MMS": write_file_names.write_mms_file_names, "NEXT_DAY_DISPATCHLOAD": write_file_names.write_file_names_current, + "INTERMITTENT_GEN_SCADA": write_file_names.write_file_names_current, "BIDDING": write_file_names.write_file_names_mms_and_current, "DAILY_REGION_SUMMARY": write_file_names.write_file_names_mms_and_current, "FCAS": write_file_names.write_file_names_fcas, @@ -261,6 +259,7 @@ downloader = { "MMS": downloader.run, "NEXT_DAY_DISPATCHLOAD": downloader.run_next_dispatch_tables, + "INTERMITTENT_GEN_SCADA": downloader.run_intermittent_gen_scada, "BIDDING": downloader.run_bid_tables, "DAILY_REGION_SUMMARY": downloader.run_next_day_region_tables, "FCAS": downloader.run_fcas4s, diff --git a/nemosis/query_wrappers.py b/src/nemosis/query_wrappers.py similarity index 98% rename from nemosis/query_wrappers.py rename to src/nemosis/query_wrappers.py index db3f451..34506c2 100644 --- a/nemosis/query_wrappers.py +++ b/src/nemosis/query_wrappers.py @@ -1,6 +1,6 @@ import pandas as pd from datetime import datetime, timedelta -from . import defaults, _parse_datetime +from nemosis import defaults, _parse_datetime def dispatch_date_setup(start_time, end_time): diff --git a/nemosis/rows.py b/src/nemosis/rows.py similarity index 90% rename from nemosis/rows.py rename to src/nemosis/rows.py index 7a8c3a8..b643e2f 100644 --- a/nemosis/rows.py +++ b/src/nemosis/rows.py @@ -1,6 +1,6 @@ import tkinter as tk import tkinter.ttk as ttk -from . import defaults +from nemosis import defaults class Query: @@ -68,8 +68,8 @@ def position(self): self.name.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.names_internal_row, + + defaults.row_height * self.row_number + + defaults.names_internal_row, column=first_sub_column, padx=padx, sticky="sw", @@ -78,8 +78,8 @@ def position(self): self.start_time_label.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.start_time_label_internal_row, + + defaults.row_height * self.row_number + + defaults.start_time_label_internal_row, column=first_sub_column, padx=defaults.standard_x_pad, sticky="sw", @@ -88,8 +88,8 @@ def position(self): self.start_time.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.start_time_internal_row, + + defaults.row_height * self.row_number + + defaults.start_time_internal_row, column=first_sub_column, padx=padx, sticky="sw", @@ -98,8 +98,8 @@ def position(self): self.end_time_label.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.end_time_label_internal_row, + + defaults.row_height * self.row_number + + defaults.end_time_label_internal_row, column=first_sub_column, padx=padx, sticky="sw", @@ -108,8 +108,8 @@ def position(self): self.end_time.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.end_time_internal_row, + + defaults.row_height * self.row_number + + defaults.end_time_internal_row, column=first_sub_column, padx=padx, sticky="sw", @@ -127,8 +127,8 @@ def position(self): self.tables.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.table_list_internal_row, + + defaults.row_height * self.row_number + + defaults.table_list_internal_row, column=second_sub_column, rowspan=defaults.list_row_span, columnspan=defaults.list_column_span, @@ -139,8 +139,8 @@ def position(self): self.delete.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.delete_button_internal_row, + + defaults.row_height * self.row_number + + defaults.delete_button_internal_row, column=defaults.last_column, sticky="nw", ) @@ -193,8 +193,8 @@ def position_column_list(self): self.col_list.grid( column=self.tables.grid_info()["column"] + defaults.list_column_span, row=defaults.query_row_offset - + self.row_number * defaults.row_height - + defaults.table_list_internal_row, + + self.row_number * defaults.row_height + + defaults.table_list_internal_row, rowspan=defaults.list_row_span, columnspan=defaults.list_column_span, padx=defaults.standard_x_pad, @@ -247,8 +247,8 @@ def position_filter_list(self): else: # Place the next filter next to the last filter. col = ( - self.filter_label[last_filter].grid_info()["column"] - + defaults.list_column_span + self.filter_label[last_filter].grid_info()["column"] + + defaults.list_column_span ) self.filter_label[column].grid( @@ -260,16 +260,16 @@ def position_filter_list(self): self.filter_label[column].update() self.filter_entry[column].grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.names_internal_row, + + defaults.row_height * self.row_number + + defaults.names_internal_row, column=col, padx=defaults.standard_x_pad, ) self.filter_entry[column].update() self.filter_list[column].grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.internal_filter_row, + + defaults.row_height * self.row_number + + defaults.internal_filter_row, column=col, columnspan=defaults.list_column_span, rowspan=defaults.list_filter_row_span, @@ -455,40 +455,40 @@ def position(self): self.merge_label.update() self.name.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.names_internal_row, + + defaults.row_height * self.row_number + + defaults.names_internal_row, column=0, padx=defaults.standard_x_pad, ) self.name.update() self.left_table_label.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.start_time_label_internal_row, + + defaults.row_height * self.row_number + + defaults.start_time_label_internal_row, column=0, padx=defaults.standard_x_pad, ) self.left_table_label.update() self.left_table.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.start_time_internal_row, + + defaults.row_height * self.row_number + + defaults.start_time_internal_row, column=0, padx=defaults.standard_x_pad, ) self.left_table.update() self.right_table_label.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.end_time_label_internal_row, + + defaults.row_height * self.row_number + + defaults.end_time_label_internal_row, column=0, padx=defaults.standard_x_pad, ) self.right_table_label.update() self.right_table.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.end_time_internal_row, + + defaults.row_height * self.row_number + + defaults.end_time_internal_row, column=0, padx=defaults.standard_x_pad, ) @@ -503,8 +503,8 @@ def position(self): self.join_types_label.update() self.join_types.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.table_list_internal_row, + + defaults.row_height * self.row_number + + defaults.table_list_internal_row, column=1, rowspan=defaults.list_row_span, columnspan=defaults.list_column_span, @@ -514,8 +514,8 @@ def position(self): self.join_types.update() self.delete.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.delete_button_internal_row, + + defaults.row_height * self.row_number + + defaults.delete_button_internal_row, column=defaults.last_column, sticky="nw", ) @@ -524,18 +524,18 @@ def position(self): label_row = defaults.query_row_offset + defaults.row_height * self.row_number label_sticky = "w" entry_row = ( - defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.names_internal_row + defaults.query_row_offset + + defaults.row_height * self.row_number + + defaults.names_internal_row ) custom_list_row = ( - defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.internal_filter_row + defaults.query_row_offset + + defaults.row_height * self.row_number + + defaults.internal_filter_row ) left_time_keys_col = ( - self.join_types.grid_info()["column"] + defaults.list_column_span + self.join_types.grid_info()["column"] + defaults.list_column_span ) self.left_time_keys_label.grid( row=label_row, @@ -561,7 +561,7 @@ def position(self): ) right_time_keys_col = ( - self.left_time_key_list.grid_info()["column"] + defaults.list_column_span + self.left_time_key_list.grid_info()["column"] + defaults.list_column_span ) self.right_time_keys_label.grid( row=label_row, @@ -587,7 +587,7 @@ def position(self): ) left_keys_col = ( - self.right_time_key_list.grid_info()["column"] + defaults.list_column_span + self.right_time_key_list.grid_info()["column"] + defaults.list_column_span ) self.left_keys_label.grid( row=label_row, @@ -613,7 +613,7 @@ def position(self): ) right_keys_col = ( - self.left_key_list.grid_info()["column"] + defaults.list_column_span + self.left_key_list.grid_info()["column"] + defaults.list_column_span ) self.right_keys_label.grid( row=label_row, @@ -786,8 +786,8 @@ def position(self): self.output_label.update() self.name.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.names_internal_row, + + defaults.row_height * self.row_number + + defaults.names_internal_row, column=0, padx=defaults.standard_x_pad, ) @@ -795,8 +795,8 @@ def position(self): self.input_label.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.start_time_label_internal_row, + + defaults.row_height * self.row_number + + defaults.start_time_label_internal_row, column=0, padx=defaults.standard_x_pad, sticky="ws", @@ -804,16 +804,16 @@ def position(self): self.input_label.update() self.input.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.start_time_internal_row, + + defaults.row_height * self.row_number + + defaults.start_time_internal_row, column=0, padx=defaults.standard_x_pad, ) self.input.update() self.delete.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.delete_button_internal_row, + + defaults.row_height * self.row_number + + defaults.delete_button_internal_row, column=defaults.last_column, sticky="nw", ) @@ -900,40 +900,40 @@ def position(self): self.merge_label.update() self.name.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.names_internal_row, + + defaults.row_height * self.row_number + + defaults.names_internal_row, column=0, padx=defaults.standard_x_pad, ) self.name.update() self.left_table_label.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.start_time_label_internal_row, + + defaults.row_height * self.row_number + + defaults.start_time_label_internal_row, column=0, padx=defaults.standard_x_pad, ) self.left_table_label.update() self.left_table.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.start_time_internal_row, + + defaults.row_height * self.row_number + + defaults.start_time_internal_row, column=0, padx=defaults.standard_x_pad, ) self.left_table.update() self.right_table_label.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.end_time_label_internal_row, + + defaults.row_height * self.row_number + + defaults.end_time_label_internal_row, column=0, padx=defaults.standard_x_pad, ) self.right_table_label.update() self.right_table.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.end_time_internal_row, + + defaults.row_height * self.row_number + + defaults.end_time_internal_row, column=0, padx=defaults.standard_x_pad, ) @@ -948,8 +948,8 @@ def position(self): self.join_types_label.update() self.join_types.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.table_list_internal_row, + + defaults.row_height * self.row_number + + defaults.table_list_internal_row, column=1, rowspan=defaults.list_row_span, columnspan=defaults.list_column_span, @@ -959,8 +959,8 @@ def position(self): self.join_types.update() self.delete.grid( row=defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.delete_button_internal_row, + + defaults.row_height * self.row_number + + defaults.delete_button_internal_row, column=defaults.last_column, sticky="nw", ) @@ -969,18 +969,18 @@ def position(self): label_row = defaults.query_row_offset + defaults.row_height * self.row_number label_sticky = "w" entry_row = ( - defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.names_internal_row + defaults.query_row_offset + + defaults.row_height * self.row_number + + defaults.names_internal_row ) custom_list_row = ( - defaults.query_row_offset - + defaults.row_height * self.row_number - + defaults.internal_filter_row + defaults.query_row_offset + + defaults.row_height * self.row_number + + defaults.internal_filter_row ) left_keys_col = ( - self.join_types.grid_info()["column"] + defaults.list_column_span + self.join_types.grid_info()["column"] + defaults.list_column_span ) self.left_keys_label.grid( row=label_row, @@ -1006,7 +1006,7 @@ def position(self): ) right_keys_col = ( - self.left_key_list.grid_info()["column"] + defaults.list_column_span + self.left_key_list.grid_info()["column"] + defaults.list_column_span ) self.right_keys_label.grid( row=label_row, diff --git a/nemosis/value_parser.py b/src/nemosis/value_parser.py similarity index 97% rename from nemosis/value_parser.py rename to src/nemosis/value_parser.py index 890c9d7..b6dc92f 100644 --- a/nemosis/value_parser.py +++ b/src/nemosis/value_parser.py @@ -1,6 +1,6 @@ import pandas as pd -from . import defaults as _defaults +from nemosis import defaults as _defaults def _parse_datetime(series): diff --git a/nemosis/write_file_names.py b/src/nemosis/write_file_names.py similarity index 95% rename from nemosis/write_file_names.py rename to src/nemosis/write_file_names.py index d28659d..b6af9ec 100644 --- a/nemosis/write_file_names.py +++ b/src/nemosis/write_file_names.py @@ -19,7 +19,7 @@ def write_file_names_mms_and_current(name, month, year, day, chunk, index, raw_d filename_stub, path_and_name = write_mms_file_names(name, month, year, day, index, chunk, raw_data_location) else: filename_stub = ( - defaults.names[name] + "_" + str(year) + str(month) + str(day) + defaults.names[name] + "_" + str(year) + str(month) + str(day) ) path_and_name = os.path.join(raw_data_location, filename_stub) return filename_stub, path_and_name @@ -28,7 +28,7 @@ def write_file_names_mms_and_current(name, month, year, day, chunk, index, raw_d def write_file_names_current(name, month, year, day, chunk, index, raw_data_location): # Add the year and month information to the generic AEMO file name filename_stub = ( - defaults.names[name] + "_" + str(year) + str(month) + str(day) + defaults.names[name] + "_" + str(year) + str(month) + str(day) ) path_and_name = os.path.join(raw_data_location, filename_stub) return filename_stub, path_and_name diff --git a/tests/test_data_fetch_methods.py b/tests/test_data_fetch_methods.py index e4cf4da..4e33d04 100644 --- a/tests/test_data_fetch_methods.py +++ b/tests/test_data_fetch_methods.py @@ -3,14 +3,10 @@ from datetime import datetime, timedelta, date from dateutil.relativedelta import relativedelta import calendar -from nemosis import data_fetch_methods -from nemosis import defaults import pandas as pd -from nemosis import custom_tables, filters +from src.nemosis import custom_tables, defaults, filters, data_fetch_methods from pandas._testing import assert_frame_equal from parameterized import parameterized -import pytz - recent_test_month = datetime.now().replace(day=1, hour=0, minute=0, second=0, microsecond=0) - relativedelta(months=2) @@ -880,11 +876,16 @@ class TestDynamicDataCompilerWithSettlementDateFilteringNextDayTables( ): def setUp(self): - self.table_names = ["DAILY_REGION_SUMMARY", "NEXT_DAY_DISPATCHLOAD"] + self.table_names = [ + "DAILY_REGION_SUMMARY", + "NEXT_DAY_DISPATCHLOAD", + "INTERMITTENT_GEN_SCADA" + ] self.table_filters = { "DAILY_REGION_SUMMARY": ["REGIONID"], "NEXT_DAY_DISPATCHLOAD": ["DUID"], + "INTERMITTENT_GEN_SCADA": ["DUID", "SCADA_TYPE"] } # Filter for bids at the start of the 2021-06-01 file and the end of the 2021-05-31, to make sure that we aren't @@ -895,6 +896,10 @@ def setUp(self): ), "NEXT_DAY_DISPATCHLOAD": ( ['AGLHAL'], + ), + "INTERMITTENT_GEN_SCADA": ( + ['ADPPV1'], + ['ELAV'] ) } diff --git a/tests/test_date_generators.py b/tests/test_date_generators.py index 741a870..28e4bf2 100644 --- a/tests/test_date_generators.py +++ b/tests/test_date_generators.py @@ -1,6 +1,6 @@ import unittest from datetime import datetime -from nemosis import date_generators +from src.nemosis import date_generators class TestYearAndMonthGen(unittest.TestCase): diff --git a/tests/test_errors_and_warnings.py b/tests/test_errors_and_warnings.py index a6f8c02..369dad2 100644 --- a/tests/test_errors_and_warnings.py +++ b/tests/test_errors_and_warnings.py @@ -1,7 +1,7 @@ import unittest -from nemosis import dynamic_data_compiler, cache_compiler, static_table, defaults +from src.nemosis import dynamic_data_compiler, cache_compiler, static_table +from src.nemosis import defaults import os -import sys from unittest.mock import patch from io import StringIO @@ -253,7 +253,7 @@ def test_raise_error_for_no_data_returned(self): good_url = defaults.static_table_url["VARIABLES_FCAS_4_SECOND"] defaults.static_table_url["VARIABLES_FCAS_4_SECOND"] = "bad_url" path_and_name = ( - defaults.raw_data_cache + "/" + defaults.names["VARIABLES_FCAS_4_SECOND"] + defaults.raw_data_cache + "/" + defaults.names["VARIABLES_FCAS_4_SECOND"] ) if os.path.isfile(path_and_name): os.remove(path_and_name) diff --git a/tests/test_filters.py b/tests/test_filters.py index af4399d..a787c5c 100644 --- a/tests/test_filters.py +++ b/tests/test_filters.py @@ -1,6 +1,6 @@ import unittest import pandas as pd -from nemosis import filters +from src.nemosis import filters from pandas.testing import assert_frame_equal from datetime import datetime import numpy as np diff --git a/tests/test_format_options.py b/tests/test_format_options.py index d1f009f..573b50a 100644 --- a/tests/test_format_options.py +++ b/tests/test_format_options.py @@ -1,7 +1,6 @@ import unittest from datetime import timedelta -from nemosis import data_fetch_methods -from nemosis import defaults +from src.nemosis import defaults, data_fetch_methods import pandas as pd import os diff --git a/tests/test_performance_stats.py b/tests/test_performance_stats.py index 70ad340..59f4579 100644 --- a/tests/test_performance_stats.py +++ b/tests/test_performance_stats.py @@ -1,11 +1,9 @@ import unittest -from nemosis import data_fetch_methods import pandas as pd -from nemosis import custom_tables +from src.nemosis import custom_tables, defaults, data_fetch_methods import math import numpy as np import time -from nemosis import defaults from datetime import datetime, timedelta import os diff --git a/tests/test_processing_info_maps.py b/tests/test_processing_info_maps.py index f260933..bff2b3c 100644 --- a/tests/test_processing_info_maps.py +++ b/tests/test_processing_info_maps.py @@ -1,12 +1,8 @@ import unittest -from soupsieve import select -from nemosis import processing_info_maps -from nemosis import data_fetch_methods -from nemosis import defaults +from src.nemosis import processing_info_maps, defaults, query_wrappers, data_fetch_methods import pandas as pd from datetime import datetime, timedelta -from nemosis import query_wrappers class TestSearchTypeValidity(unittest.TestCase): @@ -21,10 +17,10 @@ def test_start_to_end_no_duplication_between_batches(self): if processing_info_maps.search_type[table_name] == "start_to_end": print("Validating start_to_end type for table {}".format(table_name)) start_time = datetime.strptime( - "2018/01/01 00:00:00", "%Y/%m/%d %H:%M:%S" + "2018/06/01 00:00:00", "%Y/%m/%d %H:%M:%S" ) - end_time = datetime.strptime("2018/03/01 00:00:00", "%Y/%m/%d %H:%M:%S") - if table_name in ["DAILY_REGION_SUMMARY", "NEXT_DAY_DISPATCHLOAD"]: + end_time = datetime.strptime("2018/09/01 00:00:00", "%Y/%m/%d %H:%M:%S") + if table_name in ["DAILY_REGION_SUMMARY", "NEXT_DAY_DISPATCHLOAD", "INTERMITTENT_GEN_SCADA"]: end_time = self.time_yesterday start_time = self.time_yesterday - timedelta(days=8) if table_name in ["FCAS_4_SECOND"]: @@ -59,10 +55,14 @@ def test_start_to_end_has_settlement_or_interval_col(self): has_interval_timestamp_col = ( "TIMESTAMP" in defaults.table_columns[table_name] ) + has_run_datetime_col = ( + "RUN_DATETIME" in defaults.table_columns[table_name] + ) has_either = ( has_interval_datetime_col or has_settlement_date_col or has_interval_timestamp_col + or has_run_datetime_col ) self.assertEqual(True, has_either) print( @@ -202,7 +202,7 @@ def test_last_contains_data_from_first(self): ) first_data_table = data_tables[35].loc[ :, defaults.table_primary_keys[table_name] - ] + ] last_data_table = data_tables[-1] comp = pd.merge( first_data_table, diff --git a/tests/test_query_wrappers.py b/tests/test_query_wrappers.py index f1bb5e2..74f8be0 100644 --- a/tests/test_query_wrappers.py +++ b/tests/test_query_wrappers.py @@ -1,9 +1,8 @@ import unittest import pandas as pd -from nemosis import query_wrappers from pandas.testing import assert_frame_equal from datetime import datetime -from nemosis import defaults +from src.nemosis import defaults, query_wrappers class TestDispatchDateSetup(unittest.TestCase):