diff --git a/.all-contributorsrc b/.all-contributorsrc index 030978e4..ca43e082 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -81,6 +81,33 @@ "contributions": [ "doc" ] + }, + { + "login": "0xFrama", + "name": "Francesco", + "avatar_url": "https://avatars.githubusercontent.com/u/30957828?v=4", + "profile": "https://github.com/0xFrama", + "contributions": [ + "code" + ] + }, + { + "login": "roshnaeem", + "name": "Rosheen Naeem", + "avatar_url": "https://avatars.githubusercontent.com/u/47316899?v=4", + "profile": "https://github.com/roshnaeem", + "contributions": [ + "doc" + ] + }, + { + "login": "bikramb98", + "name": "Bikram Baruah", + "avatar_url": "https://avatars.githubusercontent.com/u/24806286?v=4", + "profile": "https://github.com/bikramb98", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 7, diff --git a/.bumpversion.cfg b/.bumpversion.cfg index e3c8159d..1f8d7a45 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,7 +1,7 @@ [bumpversion] commit = True tag = True -current_version = 1.0.9 +current_version = 1.0.15 message = Bump version: {current_version} β†’ {new_version} [skip ci] [bumpversion:file:setup.py] diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..b6fe388d --- /dev/null +++ b/.env.example @@ -0,0 +1,3 @@ +# User needs to add their Enphase API details +ENPHASE_API_KEY = 'user_enphase_api_key' +ENPHASE_USER_ID = 'user_enphase_user_id' \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..2eea525d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/README.md b/README.md index ff4ca0aa..d68c23fe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Quartz Solar Forecast -[![All Contributors](https://img.shields.io/badge/all_contributors-8-orange.svg?style=flat-square)](#contributors-) +[![All Contributors](https://img.shields.io/badge/all_contributors-11-orange.svg?style=flat-square)](#contributors-) The aim of the project is to build an open source PV forecast that is free and easy to use. @@ -141,6 +141,9 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d Aryan Bhosale
Aryan Bhosale
πŸ“– + Francesco
Francesco
πŸ’» + Rosheen Naeem
Rosheen Naeem
πŸ“– + Bikram Baruah
Bikram Baruah
πŸ’» diff --git a/contribution_guide.md b/contribution_guide.md new file mode 100644 index 00000000..47acca63 --- /dev/null +++ b/contribution_guide.md @@ -0,0 +1,51 @@ +# Contribution Guide + +## Welcome +Welcome to Open Climate Fix project! Open Climate Fix fosters an inclusive open-source community, welcoming participation from everyone. We value all forms of contributions, whether to our codebase or documentation, and we appreciate your support. +Before you begin, make sure to go through the [README.md](https://github.com/openclimatefix/Open-Source-Quartz-Solar-Forecast/blob/main/README.md) file of project to understand the project. + +## Environment Setup +`Open-Source-Quartz-Solar-Forecast` project is developed with Python, that supports our open-source PV forecast model's computational needs. To set up and contribute to the project, Python version 3 is required. + +### Clone the Repository +Follow the following steps to setup the project on your machine. + + +1. Fork the [Open-Source-Quartz-Solar-Forecast repository](https://github.com/openclimatefix/Open-Source-Quartz-Solar-Forecast) by clicking on the β€˜Fork’ button. This creates a copy of the code under your GitHub user account. Your fork will have your own GitHub username. + +2. Clone your fork of the project by running the following command: +``` +git clone https://github.com//Open-Source-Quartz-Solar-Forecast.git +``` + +3. Navigate to the directory. +``` +cd Open-Source-Quartz-Solar-Forecast +``` + +4. Start contributing!!! + + +## How to Contribute + +For contributing to Open Climate Fix project, we recommend following these detailed steps. + +### Step 1: Find an Issue to Work On +Begin by exploring the issues listed on the project's GitHub [issues](https://github.com/openclimatefix/Open-Source-Quartz-Solar-Forecast/issues) page. If you're new to the project, look for issues tagged with `good first issue`, these are specifically selected to be more accessible for newcomers. Selecting an issue that aligns with your interests or expertise is a great way to contribute effectively. + +### Step 2: Communicate Your Interest +Before diving into coding, it's a good practice to comment on the issue you're interested in. This lets the maintainers know you're working on it and prevents duplicate efforts. It's also an excellent opportunity to ask any clarifying questions and discuss your approach. + +### Step 3: Create a New Branch +For every new contribution, create a branch in your local repository. A branch is a way to keep your changes seprate from the main part of the project called `main`. This branch should be based off the main project's latest main branch. Naming your branch with a convention like `feature/your-feature-name` or `fix/issue-description` can help identify its purpose. + +### Step 4: Open a Pull Request (PR) +Once you're satisfied with your contribution, push your branch to your fork on GitHub and open a pull request against the original Open Climate Fix repository. In your pull request description, reference the issue you're addressing and provide a summary of your changes. Providing screenshots or output snippets can be particularly helpful for visual changes or user interface improvements. + +### Step 5: Request a Review +After submitting your pull request, you can request a review from the project maintainers or specific contributors by mentioning them in your PR comments. Once your pull request is approved, a project maintainer will merge it into the main branch. + +> Don't forget to keep your local repository and fork updated with the main project to ease future contributions. + +## Code Style +For checking our coding style, please refer to [coding_style.md](https://github.com/openclimatefix/.github/blob/main/coding_style.md). \ No newline at end of file diff --git a/quartz_solar_forecast/data.py b/quartz_solar_forecast/data.py index 4f9fce67..e4f702b0 100644 --- a/quartz_solar_forecast/data.py +++ b/quartz_solar_forecast/data.py @@ -2,6 +2,7 @@ import json import ssl from datetime import datetime +import os # Add import for os module import numpy as np import pandas as pd @@ -9,9 +10,16 @@ import xarray as xr from quartz_solar_forecast.pydantic_models import PVSite +from quartz_solar_forecast.inverters.enphase import get_enphase_data # Added import for get_enphase_data from /inverters/enphase.py ssl._create_default_https_context = ssl._create_unverified_context +# Load environment variables from .env file +from dotenv import load_dotenv + +# Assigning secrets from the .env file +ENPHASE_API_KEY = os.getenv('ENPHASE_API_KEY') +ENPHASE_USER_ID = os.getenv('ENPHASE_USER_ID') def get_nwp(site: PVSite, ts: datetime, nwp_source: str = "icon") -> xr.Dataset: """ @@ -111,20 +119,28 @@ def format_nwp_data(df: pd.DataFrame, nwp_source:str, site: PVSite): ) return data_xr - def make_pv_data(site: PVSite, ts: pd.Timestamp) -> xr.Dataset: """ - Make fake PV data for the site + Make PV data by combining Enphase live data and fake PV data Later we could add PV history here :param site: the PV site :param ts: the timestamp of the site - :return: The fake PV dataset in xarray form + :return: The combined PV dataset in xarray form """ - # make fake pv data, this is where we could add history of a pv system - generation_wh = [[np.nan]] + # Check if the site has an inverter and use_enphase_data flag accordingly + use_enphase_data = site.is_inverter + + if use_enphase_data: + # Fetch live Enphase data and store it in live_generation_wh + live_generation_wh = get_enphase_data(ENPHASE_USER_ID, ENPHASE_API_KEY) + else: + live_generation_wh = np.nan # Default value if not using live Enphase data + + # Combine live Enphase data with fake PV data, this is where we could add history of a pv system + generation_wh = [[live_generation_wh]] lon = [site.longitude] lat = [site.latitude] timestamp = [ts] @@ -145,4 +161,4 @@ def make_pv_data(site: PVSite, ts: pd.Timestamp) -> xr.Dataset: ) da = da.to_dataset(name="generation_wh") - return da + return da \ No newline at end of file diff --git a/quartz_solar_forecast/dataset/README.md b/quartz_solar_forecast/dataset/README.md new file mode 100644 index 00000000..59a8d2c1 --- /dev/null +++ b/quartz_solar_forecast/dataset/README.md @@ -0,0 +1,10 @@ +## Test set analysis +The test set contains the data used for testing the solar forecast models developed. It contains 2500 data points, with `pv_id` and `timestamps` of when the data was collected. Upon analysing the dataset, the following observations were made: +1. The data is pretty evenly distrbuted throughout the year, with the highest number of data points from the month of May (256 data points) +2. Looking at distribution by hour of the day, the highest number of data points is from 19:00 hrs (132 data points), and the least from 00:00 hrs (87 data points) + +By analysing the metadata, available at [Hugging Face](https://huggingface.co/datasets/openclimatefix/uk_pv), along with the test set, it can be observed that: +1. Most of the data in the test set has a tilt angle of 30-34 degrees +2. The maximum kwp is 4.0 & the minmum kwp is 2.25 in the test set. + +A detailed anaysis of the test set can be found at quartz_solar_forecast/dataset/dataset_analysis/test_set_analysis.ipynb \ No newline at end of file diff --git a/quartz_solar_forecast/dataset/dataset_analysis/test_set_analysis.ipynb b/quartz_solar_forecast/dataset/dataset_analysis/test_set_analysis.ipynb new file mode 100644 index 00000000..ea0053c6 --- /dev/null +++ b/quartz_solar_forecast/dataset/dataset_analysis/test_set_analysis.ipynb @@ -0,0 +1,4880 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Analysing the test set " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# To view the plots and interact with it, head to https://nbviewer.org and paste the link of this notebook" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/bikrambaruah/opt/anaconda3/envs/assignment/lib/python3.7/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import numpy as np\n", + "import pandas as pd\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline\n", + "import plotly.express as px\n", + "from huggingface_hub import hf_hub_download #used to download metadata.csv from OCF's Hugging Face repo" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pv_idtimestamp
095312021-04-14 10:45:00
195312021-10-10 13:00:00
295312021-01-28 16:45:00
395312021-05-08 10:00:00
495312021-05-08 06:45:00
\n", + "
" + ], + "text/plain": [ + " pv_id timestamp\n", + "0 9531 2021-04-14 10:45:00\n", + "1 9531 2021-10-10 13:00:00\n", + "2 9531 2021-01-28 16:45:00\n", + "3 9531 2021-05-08 10:00:00\n", + "4 9531 2021-05-08 06:45:00" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.read_csv('../testset.csv')\n", + "df.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(2500, 2)" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df.shape" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Analysing the distribution of data points by month" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + " \n", + " " + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "alignmentgroup": "True", + "hovertemplate": "Month: %{x}
Number of Data Points: %{y}", + "legendgroup": "", + "marker": { + "color": [ + 218, + 193, + 197, + 195, + 256, + 195, + 208, + 210, + 219, + 210, + 204, + 195 + ], + "coloraxis": "coloraxis", + "pattern": { + "shape": "" + } + }, + "name": "", + "offsetgroup": "", + "orientation": "v", + "showlegend": false, + "textposition": "auto", + "type": "bar", + "x": [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December" + ], + "xaxis": "x", + "y": [ + 218, + 193, + 197, + 195, + 256, + 195, + 208, + 210, + 219, + 210, + 204, + 195 + ], + "yaxis": "y" + } + ], + "layout": { + "barmode": "relative", + "coloraxis": { + "colorbar": { + "title": { + "text": "color" + } + }, + "colorscale": [ + [ + 0, + "#440154" + ], + [ + 0.1111111111111111, + "#482878" + ], + [ + 0.2222222222222222, + "#3e4989" + ], + [ + 0.3333333333333333, + "#31688e" + ], + [ + 0.4444444444444444, + "#26828e" + ], + [ + 0.5555555555555556, + "#1f9e89" + ], + [ + 0.6666666666666666, + "#35b779" + ], + [ + 0.7777777777777778, + "#6ece58" + ], + [ + 0.8888888888888888, + "#b5de2b" + ], + [ + 1, + "#fde725" + ] + ] + }, + "legend": { + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Distribution of Data Points by Month" + }, + "xaxis": { + "anchor": "y", + "categoryarray": [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December" + ], + "categoryorder": "array", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Month" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Number of Data Points" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Convert 'timestamp' column to pandas datetime type\n", + "df['timestamp'] = pd.to_datetime(df['timestamp'])\n", + "\n", + "# Extract date and time components\n", + "df['date'] = df['timestamp'].dt.date\n", + "df['time'] = df['timestamp'].dt.time\n", + "df['hour'] = df['timestamp'].dt.hour\n", + "\n", + "# Group by month and count occurrences\n", + "monthly_counts = df.groupby(df['timestamp'].dt.month).size()\n", + "hourly_counts = df.groupby(df['hour']).size().reset_index(name='count')\n", + "\n", + "# Convert month numbers to month names\n", + "monthly_counts.index = pd.to_datetime(monthly_counts.index, format='%m').month_name()\n", + "\n", + "# Plot the distribution using Plotly\n", + "fig = px.bar(x=monthly_counts.index, y=monthly_counts.values, color=monthly_counts.values,\n", + " labels={'x': 'Month', 'y': 'Number of Data Points'},\n", + " color_continuous_scale=px.colors.sequential.Viridis,\n", + " title='Distribution of Data Points by Month')\n", + "fig.update_layout(xaxis={'categoryorder': 'array', 'categoryarray': ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']})\n", + "\n", + "fig.update_traces(hovertemplate='Month: %{x}
Number of Data Points: %{y}')\n", + "\n", + "fig.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From the above, we can see that the data is pretty evenly distributed across the different months of the year, with the highest number of data points from the month of May" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Analysing the distribution of data points by hour" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "alignmentgroup": "True", + "hovertemplate": "Hour of the Day=%{x}
Number of Data Points in the Test Set=%{y}", + "legendgroup": "", + "marker": { + "color": "#636efa", + "pattern": { + "shape": "" + } + }, + "name": "", + "offsetgroup": "", + "orientation": "v", + "showlegend": false, + "textposition": "auto", + "type": "bar", + "x": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23 + ], + "xaxis": "x", + "y": [ + 87, + 90, + 110, + 108, + 100, + 106, + 103, + 97, + 106, + 103, + 106, + 98, + 109, + 93, + 119, + 109, + 107, + 97, + 93, + 132, + 115, + 99, + 103, + 110 + ], + "yaxis": "y" + } + ], + "layout": { + "barmode": "relative", + "legend": { + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Hourly Distribution of Data Points" + }, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Hour of the Day" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Number of Data Points in the Test Set" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "#Plotting the hourly distribution of data points\n", + "fig = px.bar(hourly_counts, x='hour', y='count',\n", + " labels={'hour': 'Hour of the Day', 'count': 'Number of Data Points in the Test Set'},\n", + " title='Hourly Distribution of Data Points')\n", + "fig.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From the plot above, we can see that the highest number of data points is from 19:00 hrs, and the least number of data points is from 00:00 hrs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Analysing the metadata" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The metadata.csv can be downloaded from https://huggingface.co/datasets/openclimatefix/uk_pv \n", + "
It contains a column ss_id, which is the same as pv_id in the testset.csv file\n", + "
It also contains geographical coordinates, orientation, tilt & kwp of the data points" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'../metadata.csv'" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hf_hub_download(repo_id=\"openclimatefix/uk_pv\", filename=\"metadata.csv\", repo_type=\"dataset\", local_dir='../')" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ss_idlatitude_roundedlongitude_roundedllsoacdorientationtiltkwpoperational_at
0240553.53-1.63E01007430180.035.03.362010-11-18
1240654.88-1.38E01008780315.030.01.892010-12-03
2240754.88-1.38E01008780225.030.01.892010-12-03
3240854.88-1.38E01008780225.030.01.892010-12-03
4240954.88-1.38E01008780225.030.01.892010-12-03
\n", + "
" + ], + "text/plain": [ + " ss_id latitude_rounded longitude_rounded llsoacd orientation tilt \\\n", + "0 2405 53.53 -1.63 E01007430 180.0 35.0 \n", + "1 2406 54.88 -1.38 E01008780 315.0 30.0 \n", + "2 2407 54.88 -1.38 E01008780 225.0 30.0 \n", + "3 2408 54.88 -1.38 E01008780 225.0 30.0 \n", + "4 2409 54.88 -1.38 E01008780 225.0 30.0 \n", + "\n", + " kwp operational_at \n", + "0 3.36 2010-11-18 \n", + "1 1.89 2010-12-03 \n", + "2 1.89 2010-12-03 \n", + "3 1.89 2010-12-03 \n", + "4 1.89 2010-12-03 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "metadata = pd.read_csv(\"../metadata.csv\") #loading the csv file downloaded from Hugging Face\n", + "metadata.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
ss_idlatitude_roundedlongitude_roundedllsoacdorientationtiltkwpoperational_at
689309455.74-3.85S00131230135.055.03.332011-11-06
1049345451.650.62E01021613180.045.02.852012-01-18
1084348951.04-4.23E01020299225.030.03.992012-07-06
1138354351.801.19E01022035225.043.02.662012-01-09
1459386455.08-2.96S00098293180.030.03.802012-03-06
\n", + "
" + ], + "text/plain": [ + " ss_id latitude_rounded longitude_rounded llsoacd orientation \\\n", + "689 3094 55.74 -3.85 S00131230 135.0 \n", + "1049 3454 51.65 0.62 E01021613 180.0 \n", + "1084 3489 51.04 -4.23 E01020299 225.0 \n", + "1138 3543 51.80 1.19 E01022035 225.0 \n", + "1459 3864 55.08 -2.96 S00098293 180.0 \n", + "\n", + " tilt kwp operational_at \n", + "689 55.0 3.33 2011-11-06 \n", + "1049 45.0 2.85 2012-01-18 \n", + "1084 30.0 3.99 2012-07-06 \n", + "1138 43.0 2.66 2012-01-09 \n", + "1459 30.0 3.80 2012-03-06 " + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Creating a new pandas df testdata_metadata, which only contains points where the ss_id matches pv_id\n", + "testdata_metadata = metadata[metadata['ss_id'].isin(df['pv_id'])]\n", + "testdata_metadata.head()" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "alignmentgroup": "True", + "bingroup": "x", + "hovertemplate": "Test Data Tilt Angle (degrees)=%{x}
count=%{y}", + "legendgroup": "", + "marker": { + "color": "#636efa", + "pattern": { + "shape": "" + } + }, + "name": "", + "nbinsx": 10, + "offsetgroup": "", + "orientation": "v", + "showlegend": false, + "type": "histogram", + "x": [ + 55, + 45, + 30, + 43, + 30, + 30, + 40, + 30, + 30, + 30, + 35, + 35, + 45, + 35, + 45, + 32, + 45, + 30, + 32, + 35, + 30, + 30, + 40, + 30, + 35, + 35, + 35, + 30, + 35, + 30, + 20, + 40, + 40, + 35, + 30, + 41, + 35, + 23, + 34, + 30, + 33, + 30, + 37, + 45, + 36, + 30, + 43, + 35, + 32, + 30 + ], + "xaxis": "x", + "yaxis": "y" + } + ], + "layout": { + "barmode": "relative", + "legend": { + "tracegroupgap": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "title": { + "text": "Tilt Angle Histogram" + }, + "xaxis": { + "anchor": "y", + "domain": [ + 0, + 1 + ], + "title": { + "text": "Test Data Tilt Angle (degrees)" + } + }, + "yaxis": { + "anchor": "x", + "domain": [ + 0, + 1 + ], + "title": { + "text": "count" + } + } + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Plot the histogram of tilt angles using Plotly\n", + "fig = px.histogram(testdata_metadata, x='tilt', nbins=10, labels={'tilt': 'Test Data Tilt Angle (degrees)'}, title='Tilt Angle Histogram')\n", + "fig.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that most of the data in the test set has a tilt of 30-34 degrees " + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.plotly.v1+json": { + "config": { + "plotlyServerURL": "https://plot.ly" + }, + "data": [ + { + "customdata": [ + [ + 3.33 + ], + [ + 2.85 + ], + [ + 3.99 + ], + [ + 2.66 + ], + [ + 3.8 + ], + [ + 3.84 + ], + [ + 2.7 + ], + [ + 4 + ], + [ + 3.04 + ], + [ + 3.84 + ], + [ + 2.5 + ], + [ + 4 + ], + [ + 4 + ], + [ + 4 + ], + [ + 3.99 + ], + [ + 3.92 + ], + [ + 4 + ], + [ + 2.55 + ], + [ + 3.92 + ], + [ + 3.92 + ], + [ + 2.55 + ], + [ + 4 + ], + [ + 4 + ], + [ + 3 + ], + [ + 3 + ], + [ + 4 + ], + [ + 4 + ], + [ + 3 + ], + [ + 4 + ], + [ + 3 + ], + [ + 4 + ], + [ + 3.25 + ], + [ + 3 + ], + [ + 2.75 + ], + [ + 2.5 + ], + [ + 3 + ], + [ + 3 + ], + [ + 4 + ], + [ + 2.5 + ], + [ + 4 + ], + [ + 2.5 + ], + [ + 2.28 + ], + [ + 2.25 + ], + [ + 2.5 + ], + [ + 4 + ], + [ + 3.15 + ], + [ + 3.33 + ], + [ + 4 + ], + [ + 2.5 + ], + [ + 2.94 + ] + ], + "hovertemplate": "%{hovertext}

kwp=%{marker.color}
latitude_rounded=%{lat}
longitude_rounded=%{lon}", + "hovertext": [ + 3.33, + 2.85, + 3.99, + 2.66, + 3.8, + 3.84, + 2.7, + 4, + 3.04, + 3.84, + 2.5, + 4, + 4, + 4, + 3.99, + 3.92, + 4, + 2.55, + 3.92, + 3.92, + 2.55, + 4, + 4, + 3, + 3, + 4, + 4, + 3, + 4, + 3, + 4, + 3.25, + 3, + 2.75, + 2.5, + 3, + 3, + 4, + 2.5, + 4, + 2.5, + 2.28, + 2.25, + 2.5, + 4, + 3.15, + 3.33, + 4, + 2.5, + 2.94 + ], + "lat": [ + 55.74, + 51.65, + 51.04, + 51.8, + 55.08, + 55.64, + 52.92, + 53.76, + 55.83, + 51.03, + 53.7, + 56.18, + 53.14, + 53.76, + 54.01, + 54.19, + 50.54, + 55.8, + 55.06, + 57.58, + 55.68, + 50.83, + 51.73, + 54.31, + 51.52, + 50.43, + 50.47, + 53.67, + 50.04, + 57.44, + 50.85, + 50.39, + 51.65, + 54.96, + 51.7, + 55.78, + 55.68, + 53.38, + 53.35, + 51.41, + 53.44, + 51.56, + 53.52, + 51.81, + 51.26, + 52.65, + 51.3, + 51.5, + 53.49, + 51.49 + ], + "legendgroup": "", + "lon": [ + -3.85, + 0.62, + -4.23, + 1.19, + -2.96, + -3.89, + -1.21, + -1.45, + -4.29, + -2.22, + -1.72, + -3.16, + -1.17, + -1.23, + -1.46, + -0.93, + -4.14, + -4.38, + -1.46, + -2.33, + -3.76, + -2.31, + 0.9, + -2.72, + -2.57, + -3.67, + -3.56, + -1.79, + -5.12, + -3.12, + -4.14, + -4.13, + -3.24, + -1.43, + -3.01, + -3.56, + -3.78, + -3.03, + -3.11, + -1.76, + -2.2, + -1.83, + -2.32, + -0.77, + -0.61, + 1.23, + -0.57, + -2.58, + -2.87, + -2.5 + ], + "marker": { + "color": [ + 3.33, + 2.85, + 3.99, + 2.66, + 3.8, + 3.84, + 2.7, + 4, + 3.04, + 3.84, + 2.5, + 4, + 4, + 4, + 3.99, + 3.92, + 4, + 2.55, + 3.92, + 3.92, + 2.55, + 4, + 4, + 3, + 3, + 4, + 4, + 3, + 4, + 3, + 4, + 3.25, + 3, + 2.75, + 2.5, + 3, + 3, + 4, + 2.5, + 4, + 2.5, + 2.28, + 2.25, + 2.5, + 4, + 3.15, + 3.33, + 4, + 2.5, + 2.94 + ], + "coloraxis": "coloraxis", + "size": [ + 3.33, + 2.85, + 3.99, + 2.66, + 3.8, + 3.84, + 2.7, + 4, + 3.04, + 3.84, + 2.5, + 4, + 4, + 4, + 3.99, + 3.92, + 4, + 2.55, + 3.92, + 3.92, + 2.55, + 4, + 4, + 3, + 3, + 4, + 4, + 3, + 4, + 3, + 4, + 3.25, + 3, + 2.75, + 2.5, + 3, + 3, + 4, + 2.5, + 4, + 2.5, + 2.28, + 2.25, + 2.5, + 4, + 3.15, + 3.33, + 4, + 2.5, + 2.94 + ], + "sizemode": "area", + "sizeref": 0.01 + }, + "mode": "markers", + "name": "", + "showlegend": false, + "subplot": "mapbox", + "type": "scattermapbox" + } + ], + "layout": { + "coloraxis": { + "colorbar": { + "title": { + "text": "kwp" + } + }, + "colorscale": [ + [ + 0, + "orange" + ], + [ + 1, + "red" + ] + ] + }, + "height": 800, + "legend": { + "itemsizing": "constant", + "tracegroupgap": 0 + }, + "mapbox": { + "center": { + "lat": 53.114799999999995, + "lon": -2.364 + }, + "domain": { + "x": [ + 0, + 1 + ], + "y": [ + 0, + 1 + ] + }, + "style": "open-street-map", + "zoom": 8 + }, + "margin": { + "b": 0, + "l": 0, + "r": 0, + "t": 0 + }, + "template": { + "data": { + "bar": [ + { + "error_x": { + "color": "#2a3f5f" + }, + "error_y": { + "color": "#2a3f5f" + }, + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "bar" + } + ], + "barpolar": [ + { + "marker": { + "line": { + "color": "#E5ECF6", + "width": 0.5 + }, + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "barpolar" + } + ], + "carpet": [ + { + "aaxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "baxis": { + "endlinecolor": "#2a3f5f", + "gridcolor": "white", + "linecolor": "white", + "minorgridcolor": "white", + "startlinecolor": "#2a3f5f" + }, + "type": "carpet" + } + ], + "choropleth": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "choropleth" + } + ], + "contour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "contour" + } + ], + "contourcarpet": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "contourcarpet" + } + ], + "heatmap": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmap" + } + ], + "heatmapgl": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "heatmapgl" + } + ], + "histogram": [ + { + "marker": { + "pattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + } + }, + "type": "histogram" + } + ], + "histogram2d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2d" + } + ], + "histogram2dcontour": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "histogram2dcontour" + } + ], + "mesh3d": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "type": "mesh3d" + } + ], + "parcoords": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "parcoords" + } + ], + "pie": [ + { + "automargin": true, + "type": "pie" + } + ], + "scatter": [ + { + "fillpattern": { + "fillmode": "overlay", + "size": 10, + "solidity": 0.2 + }, + "type": "scatter" + } + ], + "scatter3d": [ + { + "line": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatter3d" + } + ], + "scattercarpet": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattercarpet" + } + ], + "scattergeo": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergeo" + } + ], + "scattergl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattergl" + } + ], + "scattermapbox": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scattermapbox" + } + ], + "scatterpolar": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolar" + } + ], + "scatterpolargl": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterpolargl" + } + ], + "scatterternary": [ + { + "marker": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "type": "scatterternary" + } + ], + "surface": [ + { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + }, + "colorscale": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "type": "surface" + } + ], + "table": [ + { + "cells": { + "fill": { + "color": "#EBF0F8" + }, + "line": { + "color": "white" + } + }, + "header": { + "fill": { + "color": "#C8D4E3" + }, + "line": { + "color": "white" + } + }, + "type": "table" + } + ] + }, + "layout": { + "annotationdefaults": { + "arrowcolor": "#2a3f5f", + "arrowhead": 0, + "arrowwidth": 1 + }, + "autotypenumbers": "strict", + "coloraxis": { + "colorbar": { + "outlinewidth": 0, + "ticks": "" + } + }, + "colorscale": { + "diverging": [ + [ + 0, + "#8e0152" + ], + [ + 0.1, + "#c51b7d" + ], + [ + 0.2, + "#de77ae" + ], + [ + 0.3, + "#f1b6da" + ], + [ + 0.4, + "#fde0ef" + ], + [ + 0.5, + "#f7f7f7" + ], + [ + 0.6, + "#e6f5d0" + ], + [ + 0.7, + "#b8e186" + ], + [ + 0.8, + "#7fbc41" + ], + [ + 0.9, + "#4d9221" + ], + [ + 1, + "#276419" + ] + ], + "sequential": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ], + "sequentialminus": [ + [ + 0, + "#0d0887" + ], + [ + 0.1111111111111111, + "#46039f" + ], + [ + 0.2222222222222222, + "#7201a8" + ], + [ + 0.3333333333333333, + "#9c179e" + ], + [ + 0.4444444444444444, + "#bd3786" + ], + [ + 0.5555555555555556, + "#d8576b" + ], + [ + 0.6666666666666666, + "#ed7953" + ], + [ + 0.7777777777777778, + "#fb9f3a" + ], + [ + 0.8888888888888888, + "#fdca26" + ], + [ + 1, + "#f0f921" + ] + ] + }, + "colorway": [ + "#636efa", + "#EF553B", + "#00cc96", + "#ab63fa", + "#FFA15A", + "#19d3f3", + "#FF6692", + "#B6E880", + "#FF97FF", + "#FECB52" + ], + "font": { + "color": "#2a3f5f" + }, + "geo": { + "bgcolor": "white", + "lakecolor": "white", + "landcolor": "#E5ECF6", + "showlakes": true, + "showland": true, + "subunitcolor": "white" + }, + "hoverlabel": { + "align": "left" + }, + "hovermode": "closest", + "mapbox": { + "style": "light" + }, + "paper_bgcolor": "white", + "plot_bgcolor": "#E5ECF6", + "polar": { + "angularaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "radialaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "scene": { + "xaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "yaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + }, + "zaxis": { + "backgroundcolor": "#E5ECF6", + "gridcolor": "white", + "gridwidth": 2, + "linecolor": "white", + "showbackground": true, + "ticks": "", + "zerolinecolor": "white" + } + }, + "shapedefaults": { + "line": { + "color": "#2a3f5f" + } + }, + "ternary": { + "aaxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "baxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + }, + "bgcolor": "#E5ECF6", + "caxis": { + "gridcolor": "white", + "linecolor": "white", + "ticks": "" + } + }, + "title": { + "x": 0.05 + }, + "xaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + }, + "yaxis": { + "automargin": true, + "gridcolor": "white", + "linecolor": "white", + "ticks": "", + "title": { + "standoff": 15 + }, + "zerolinecolor": "white", + "zerolinewidth": 2 + } + } + }, + "width": 800 + } + }, + "text/html": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Visualizing test set points on a map, distinguished by a gradient color scheme representing kwp values.\n", + "\n", + "df.dropna(\n", + " axis=0,\n", + " how='any',\n", + " thresh=None,\n", + " subset=None,\n", + " inplace=True\n", + ")\n", + "\n", + "color_scale = [(0, 'orange'), (1,'red')]\n", + "\n", + "fig = px.scatter_mapbox(testdata_metadata, \n", + " lat=\"latitude_rounded\", \n", + " lon=\"longitude_rounded\", \n", + " hover_name=\"kwp\", \n", + " hover_data=[\"kwp\"],\n", + " color=\"kwp\",\n", + " color_continuous_scale=color_scale,\n", + " size=\"kwp\",\n", + " zoom=8, \n", + " height=800,\n", + " width=800)\n", + "\n", + "fig.update_layout(mapbox_style=\"open-street-map\")\n", + "fig.update_layout(margin={\"r\":0,\"t\":0,\"l\":0,\"b\":0})\n", + "fig.show()" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "In the test set, the maximum kwp is 4.0 & the minimum kwp is 2.25\n" + ] + } + ], + "source": [ + "max_kwp = np.max(testdata_metadata['kwp'])\n", + "min_kwp = np.min(testdata_metadata['kwp'])\n", + "\n", + "print(f\"In the test set, the maximum kwp is {max_kwp} & the minimum kwp is {min_kwp}\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.9" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/quartz_solar_forecast/dataset/make_test_set.py b/quartz_solar_forecast/dataset/make_test_set.py index eb4cf0b3..d96a6c07 100644 --- a/quartz_solar_forecast/dataset/make_test_set.py +++ b/quartz_solar_forecast/dataset/make_test_set.py @@ -116,7 +116,7 @@ def filter_timestamps_if_hf_files_exists(timestamps_full: pd.DatetimeIndex): fs = HfFileSystem() # print(fs.ls("datasets/openclimatefix/dwd-icon-eu/data/2022/4/11/", detail=False)) for timestamp in timestamps_full: - timestamp_floor = timestamp.floor("6H") + timestamp_floor = timestamp.floor("6h") _, huggingface_file = make_hf_filename(timestamp_floor) huggingface_file = huggingface_file[14:] diff --git a/quartz_solar_forecast/eval/nwp.py b/quartz_solar_forecast/eval/nwp.py index 1edc3a20..ac32f59d 100644 --- a/quartz_solar_forecast/eval/nwp.py +++ b/quartz_solar_forecast/eval/nwp.py @@ -81,7 +81,7 @@ def get_nwp_for_one_timestamp_one_location( # print(fs.ls("datasets/openclimatefix/dwd-icon-eu/data/2022/4/11/", detail=False)) # round timestamp to 6 hours floor - timestamp_floor = timestamp.floor("6H") + timestamp_floor = timestamp.floor("6h") date_and_hour, huggingface_file = make_hf_filename(timestamp_floor) # dataset variables, note these are unique for ICON diff --git a/quartz_solar_forecast/inverters/__init__.py b/quartz_solar_forecast/inverters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/quartz_solar_forecast/inverters/enphase.py b/quartz_solar_forecast/inverters/enphase.py new file mode 100644 index 00000000..dc25a687 --- /dev/null +++ b/quartz_solar_forecast/inverters/enphase.py @@ -0,0 +1,20 @@ +import requests + +def get_enphase_data(enphase_user_id: str, enphase_api_key: str) -> float: + """ + Get live PV generation data from Enphase API + + :param enphase_user_id: User ID for Enphase API + :param enphase_api_key: API Key for Enphase API + :return: Live PV generation in Watt-hours, assumes to be a floating-point number + """ + url = f'https://api.enphaseenergy.com/api/v2/systems/{enphase_user_id}/summary' + headers = {'Authorization': f'Bearer {enphase_api_key}'} + + response = requests.get(url, headers=headers) + data = response.json() + + # Extracting live generation data assuming it's in Watt-hours + live_generation_wh = data['current_power']['power'] + + return live_generation_wh \ No newline at end of file diff --git a/quartz_solar_forecast/pydantic_models.py b/quartz_solar_forecast/pydantic_models.py index 4e626bfb..260c264d 100644 --- a/quartz_solar_forecast/pydantic_models.py +++ b/quartz_solar_forecast/pydantic_models.py @@ -17,3 +17,7 @@ class PVSite(BaseModel): ge=0, le=360, ) + is_inverter: bool = Field( + default=False, + description="Flag indicating whether the site has an inverter (e.g., Enphase). If True, use_enphase_data will be considered.", + ) \ No newline at end of file diff --git a/setup.py b/setup.py index 9ffe6fd2..fdb51748 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name="quartz_solar_forecast", - version="1.0.9", + version="1.0.15", license="MIT", author="Peter Dudfield", author_email="info@openclimatefix.org",