From c1e6adcdb5d8533650d57cf2dd035ba6ce7dc19f Mon Sep 17 00:00:00 2001 From: Kiril Maneichyk Date: Tue, 22 Aug 2023 20:25:27 +0200 Subject: [PATCH 1/8] Fix failing API request --- parsers/UA.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/parsers/UA.py b/parsers/UA.py index 19dc1e8c82..18d0f31c6d 100644 --- a/parsers/UA.py +++ b/parsers/UA.py @@ -1,12 +1,13 @@ #!/usr/bin/env python3 +import http.client +import json from datetime import datetime from logging import Logger, getLogger from typing import Optional import arrow import dateutil -from requests import Session """ tec - same as `tes` but also working as central heater, @@ -38,24 +39,26 @@ def fetch_production( zone_key: str = "UA", - session: Optional[Session] = None, target_datetime: Optional[datetime] = None, logger: Logger = getLogger(__name__), ) -> list: if target_datetime: raise NotImplementedError("This parser is not yet able to parse past dates") - r = session or Session() data = [] today = arrow.now(tz=tz).format("DD.MM.YYYY") - url = "https://ua.energy/wp-admin/admin-ajax.php" - postdata = {"action": "get_data_oes", "report_date": today, "type": "day"} - response = r.post( - url, postdata, headers={"User-Agent": "electricitymap-parser/1.0"} - ) - - for serie in response.json(): + conn = http.client.HTTPSConnection("ua.energy") + payload = f"action=get_data_oes&report_date={today}&type=day" + headers = { + "Content-Type": "application/x-www-form-urlencoded", + "User-Agent": "PostmanRuntime/7.32.3", + } + conn.request("POST", "/wp-admin/admin-ajax.php", payload, headers) + res = conn.getresponse() + response = json.loads(res.read().decode("utf-8")) + + for serie in response: row = { "zoneKey": zone_key, "production": {}, @@ -75,6 +78,10 @@ def fetch_production( row["production"][v] = 0.0 # Date + # For some reason, every hour returned normally as string, except for 12 AM + if serie["hour"] == 24: + serie["hour"] = "24:00" + date = arrow.get("%s %s" % (today, serie["hour"]), "DD.MM.YYYY HH:mm") row["datetime"] = date.replace(tzinfo=dateutil.tz.gettz(tz)).datetime From 1f9229e6e33086d385eff950d3f7b35f96d5e2ae Mon Sep 17 00:00:00 2001 From: Kiril Maneichyk Date: Tue, 22 Aug 2023 20:28:27 +0200 Subject: [PATCH 2/8] Add support for past dates --- parsers/UA.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/parsers/UA.py b/parsers/UA.py index 18d0f31c6d..0144b15999 100644 --- a/parsers/UA.py +++ b/parsers/UA.py @@ -43,13 +43,14 @@ def fetch_production( logger: Logger = getLogger(__name__), ) -> list: if target_datetime: - raise NotImplementedError("This parser is not yet able to parse past dates") + target_date = arrow.get(target_datetime.date()).format("DD.MM.YYYY") + else: + target_date = arrow.now().format("DD.MM.YYYY") data = [] - today = arrow.now(tz=tz).format("DD.MM.YYYY") conn = http.client.HTTPSConnection("ua.energy") - payload = f"action=get_data_oes&report_date={today}&type=day" + payload = f"action=get_data_oes&report_date={target_date}&type=day" headers = { "Content-Type": "application/x-www-form-urlencoded", "User-Agent": "PostmanRuntime/7.32.3", @@ -82,10 +83,17 @@ def fetch_production( if serie["hour"] == 24: serie["hour"] = "24:00" - date = arrow.get("%s %s" % (today, serie["hour"]), "DD.MM.YYYY HH:mm") + date = arrow.get(f"{target_date} {serie['hour']}", "DD.MM.YYYY HH:mm") row["datetime"] = date.replace(tzinfo=dateutil.tz.gettz(tz)).datetime - data.append(row) + if target_datetime: + target_time = arrow.get(target_datetime).format("HH:mm") + if target_time == serie["hour"]: + data.append(row) + return data + else: + data.append(row) + return data From aaf83cd1756540f4bbde016658eb0f3bb08ae1a0 Mon Sep 17 00:00:00 2001 From: Kiril Maneichyk Date: Wed, 23 Aug 2023 21:27:21 +0200 Subject: [PATCH 3/8] Restore signature --- parsers/UA.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parsers/UA.py b/parsers/UA.py index 0144b15999..7fc7137ca4 100644 --- a/parsers/UA.py +++ b/parsers/UA.py @@ -8,6 +8,7 @@ import arrow import dateutil +from requests import Session """ tec - same as `tes` but also working as central heater, @@ -39,6 +40,7 @@ def fetch_production( zone_key: str = "UA", + session: Optional[Session] = None, target_datetime: Optional[datetime] = None, logger: Logger = getLogger(__name__), ) -> list: From 2b9ccbeaf48f77d24ac4c9b4dd7aa050368468d8 Mon Sep 17 00:00:00 2001 From: Kiril Maneichyk Date: Wed, 23 Aug 2023 21:42:30 +0200 Subject: [PATCH 4/8] Date fixes --- parsers/UA.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parsers/UA.py b/parsers/UA.py index 7fc7137ca4..77b93ca8b7 100644 --- a/parsers/UA.py +++ b/parsers/UA.py @@ -45,9 +45,9 @@ def fetch_production( logger: Logger = getLogger(__name__), ) -> list: if target_datetime: - target_date = arrow.get(target_datetime.date()).format("DD.MM.YYYY") + target_date = target_datetime.date().strftime("%d.%m.%Y") else: - target_date = arrow.now().format("DD.MM.YYYY") + target_date = arrow.now(tz=tz).strftime("%d.%m.%Y") data = [] From eed92477623de79ac1fe52cbaaed1e2c0bea30cb Mon Sep 17 00:00:00 2001 From: Kiril Maneichyk Date: Thu, 24 Aug 2023 08:55:09 +0200 Subject: [PATCH 5/8] Remove filtering by time. Return whole day of data --- parsers/UA.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/parsers/UA.py b/parsers/UA.py index 77b93ca8b7..0aeea55b21 100644 --- a/parsers/UA.py +++ b/parsers/UA.py @@ -88,13 +88,7 @@ def fetch_production( date = arrow.get(f"{target_date} {serie['hour']}", "DD.MM.YYYY HH:mm") row["datetime"] = date.replace(tzinfo=dateutil.tz.gettz(tz)).datetime - if target_datetime: - target_time = arrow.get(target_datetime).format("HH:mm") - if target_time == serie["hour"]: - data.append(row) - return data - else: - data.append(row) + data.append(row) return data From 78cf3d06d10acc57fe122513d09809c3422e5567 Mon Sep 17 00:00:00 2001 From: Kiril Maneichyk Date: Thu, 24 Aug 2023 09:05:26 +0200 Subject: [PATCH 6/8] Add comment regarding usage of http.client --- parsers/UA.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parsers/UA.py b/parsers/UA.py index 0aeea55b21..1a1fd6ebb0 100644 --- a/parsers/UA.py +++ b/parsers/UA.py @@ -51,6 +51,8 @@ def fetch_production( data = [] + # We are using HTTP.client because Request returns 403 http codes. + # TODO: Look into why requests are returning 403 http codes while HTTP.client works. conn = http.client.HTTPSConnection("ua.energy") payload = f"action=get_data_oes&report_date={target_date}&type=day" headers = { From e5ad0df9226b324212939b3117888d3bba3a2a5c Mon Sep 17 00:00:00 2001 From: Viktor Andersson <30777521+VIKTORVAV99@users.noreply.github.com> Date: Thu, 24 Aug 2023 10:55:39 +0200 Subject: [PATCH 7/8] ProductionBreakDownList and datetime over arrow --- parsers/UA.py | 86 ++++++++++++++++++++++++++++----------------------- 1 file changed, 48 insertions(+), 38 deletions(-) diff --git a/parsers/UA.py b/parsers/UA.py index 1a1fd6ebb0..4890855e07 100644 --- a/parsers/UA.py +++ b/parsers/UA.py @@ -2,14 +2,17 @@ import http.client import json -from datetime import datetime +from datetime import datetime, timedelta from logging import Logger, getLogger from typing import Optional -import arrow -import dateutil +from pytz import timezone from requests import Session +from electricitymap.contrib.lib.models.event_lists import ProductionBreakdownList +from electricitymap.contrib.lib.models.events import ProductionMix, StorageMix +from electricitymap.contrib.lib.types import ZoneKey + """ tec - same as `tes` but also working as central heater, main fuel is gas, in critical situations - black oil @@ -23,38 +26,37 @@ "tec": "gas", "tes": "coal", "vde": "unknown", - "biomass": "biomass", "gesgaes": "hydro", - "solar": "solar", - "wind": "wind", - "oil": "oil", - "geothermal": "geothermal", } MAP_STORAGE = { "consumptiongaespump": "hydro", } -tz = "Europe/Kiev" +IGNORED_VALUES = ["hour", "consumption", "consumption_diff"] + +SOURCE = "ua.energy" + +TZ = timezone("Europe/Kiev") def fetch_production( - zone_key: str = "UA", + zone_key: ZoneKey = ZoneKey("UA"), session: Optional[Session] = None, target_datetime: Optional[datetime] = None, logger: Logger = getLogger(__name__), ) -> list: if target_datetime: - target_date = target_datetime.date().strftime("%d.%m.%Y") + target_datetime = target_datetime.astimezone(TZ) else: - target_date = arrow.now(tz=tz).strftime("%d.%m.%Y") + target_datetime = datetime.now(TZ) data = [] # We are using HTTP.client because Request returns 403 http codes. # TODO: Look into why requests are returning 403 http codes while HTTP.client works. conn = http.client.HTTPSConnection("ua.energy") - payload = f"action=get_data_oes&report_date={target_date}&type=day" + payload = f"action=get_data_oes&report_date={target_datetime.date().strftime('%d.%m.%Y')}&type=day" headers = { "Content-Type": "application/x-www-form-urlencoded", "User-Agent": "PostmanRuntime/7.32.3", @@ -63,36 +65,44 @@ def fetch_production( res = conn.getresponse() response = json.loads(res.read().decode("utf-8")) + print(response) + + production_list = ProductionBreakdownList(logger) + for serie in response: - row = { - "zoneKey": zone_key, - "production": {}, - "storage": {}, - "source": "ua.energy", - } - - # Storage - if "consumptiongaespump" in serie: - row["storage"]["hydro"] = serie["consumptiongaespump"] * -1 - - # Production - for k, v in MAP_GENERATION.items(): - if k in serie: - row["production"][v] = serie[k] - else: - row["production"][v] = 0.0 + production = ProductionMix() + storage = StorageMix() + + for mode in serie: + # Production + if mode in MAP_GENERATION: + production.add_value(MAP_GENERATION[mode], serie[mode]) + # Storage + elif mode in MAP_STORAGE: + storage.add_value(MAP_STORAGE[mode], -serie[mode]) + # Log unknown modes + elif mode not in IGNORED_VALUES: + logger.warning(f"Unknown mode: {mode}") # Date # For some reason, every hour returned normally as string, except for 12 AM if serie["hour"] == 24: - serie["hour"] = "24:00" - - date = arrow.get(f"{target_date} {serie['hour']}", "DD.MM.YYYY HH:mm") - row["datetime"] = date.replace(tzinfo=dateutil.tz.gettz(tz)).datetime - - data.append(row) - - return data + target_datetime = target_datetime + timedelta(days=1) + serie["hour"] = "00:00" + + date_time = target_datetime.replace( + hour=int(serie["hour"][:2]), minute=0, second=0, microsecond=0 + ) + + production_list.append( + zoneKey=zone_key, + production=production, + storage=storage, + datetime=date_time, + source=SOURCE, + ) + + return production_list.to_list() if __name__ == "__main__": From 1632d22834172205e15eedaef0eeefe28202196e Mon Sep 17 00:00:00 2001 From: Kiril Maneichyk Date: Thu, 24 Aug 2023 14:41:13 +0200 Subject: [PATCH 8/8] Remove not used code --- parsers/UA.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/parsers/UA.py b/parsers/UA.py index 4890855e07..6ea7abdd28 100644 --- a/parsers/UA.py +++ b/parsers/UA.py @@ -51,8 +51,6 @@ def fetch_production( else: target_datetime = datetime.now(TZ) - data = [] - # We are using HTTP.client because Request returns 403 http codes. # TODO: Look into why requests are returning 403 http codes while HTTP.client works. conn = http.client.HTTPSConnection("ua.energy") @@ -65,8 +63,6 @@ def fetch_production( res = conn.getresponse() response = json.loads(res.read().decode("utf-8")) - print(response) - production_list = ProductionBreakdownList(logger) for serie in response: