From da70910e896da4894c1e1fc53d82f0195628bd0a Mon Sep 17 00:00:00 2001 From: Peter Mitri Date: Fri, 4 Oct 2024 14:45:54 +0200 Subject: [PATCH] Cucumber: add short tests & refactor some steps (#2382) --- .../cucumber/features/medium_tests.feature | 2 +- .../cucumber/features/short_tests.feature | 110 ++++++++++++++++-- .../cucumber/features/steps/assertions.py | 3 +- .../cucumber/features/steps/context_utils.py | 22 ---- .../cucumber/features/steps/output_utils.py | 37 ------ .../features/steps/simulator_utils.py | 24 ++-- src/tests/cucumber/features/steps/steps.py | 108 +++++++++++++---- .../features/steps/study_input_handler.py | 32 ++++- .../features/steps/study_output_handler.py | 84 +++++++++++++ src/tests/cucumber/requirements.txt | 3 +- 10 files changed, 316 insertions(+), 109 deletions(-) delete mode 100644 src/tests/cucumber/features/steps/context_utils.py delete mode 100644 src/tests/cucumber/features/steps/output_utils.py create mode 100644 src/tests/cucumber/features/steps/study_output_handler.py diff --git a/src/tests/cucumber/features/medium_tests.feature b/src/tests/cucumber/features/medium_tests.feature index 98e8a6fb26..fd11932be4 100644 --- a/src/tests/cucumber/features/medium_tests.feature +++ b/src/tests/cucumber/features/medium_tests.feature @@ -4,7 +4,7 @@ Feature: medium tests Scenario: 035 Mixed Expansion - Smart grid model 2 Given the study path is "medium-tests/035 Mixed Expansion - Smart grid model 2" When I run antares simulator - Then the simulation takes less than 15 seconds + Then the simulation takes less than 30 seconds And the simulation succeeds And the expected value of the annual system cost is 3.725e+10 And the minimum annual system cost is 3.642e+10 diff --git a/src/tests/cucumber/features/short_tests.feature b/src/tests/cucumber/features/short_tests.feature index 8a1d65d6c7..cb6a9cebbc 100644 --- a/src/tests/cucumber/features/short_tests.feature +++ b/src/tests/cucumber/features/short_tests.feature @@ -6,9 +6,7 @@ Feature: short tests When I run antares simulator Then the simulation takes less than 5 seconds And the simulation succeeds - And the annual system cost is - | EXP | STD | MIN | MAX | - | 0 | 0 | 0 | 0 | + And the annual system cost is 0 @fast @short Scenario: 002 Thermal fleet - Base @@ -16,11 +14,9 @@ Feature: short tests When I run antares simulator Then the simulation takes less than 5 seconds And the simulation succeeds - And the annual system cost is - | EXP | STD | MIN | MAX | - | 2.729e+7 | 0 | 2.729e+7 | 2.729e+7 | + And the annual system cost is 2.729e+7 And in area "AREA", during year 1, loss of load lasts 1 hours - And in area "AREA", unsupplied energy on "02 JAN 09:00" of year 1 is of 52 MW + And in area "AREA", unsupplied energy on "2 JAN 09:00" of year 1 is of 52 MW @fast @short Scenario: 003 Thermal fleet - Must-run @@ -28,11 +24,103 @@ Feature: short tests When I run antares simulator Then the simulation takes less than 5 seconds And the simulation succeeds - And the annual system cost is - | EXP | STD | MIN | MAX | - | 2.751e+7 | 0 | 2.751e+7 | 2.751e+7 | + And the annual system cost is 2.751e+7 + And in area "AREA", during year 1, loss of load lasts 1 hours + And in area "AREA", unsupplied energy on "2 JAN 09:00" of year 1 is of 52 MW + And in area "AREA", during year 1, hourly production of "non-dispatchable semi base" is always equal to 300 MWh + + @fast @short + Scenario: 004 Thermal fleet - Partial must-run + Given the study path is "short-tests/004 Thermal fleet - Partial must-run" + When I run antares simulator + Then the simulation takes less than 5 seconds + And the simulation succeeds + And the annual system cost is 2.751e+7 + And in area "AREA", during year 1, loss of load lasts 1 hours + And in area "AREA", unsupplied energy on "2 JAN 09:00" of year 1 is of 52 MW + And in area "AREA", during year 1, hourly production of "semi base" is always greater than 300 MWh + + @fast @short + Scenario: 005 Thermal fleet - Minimum stable power and min up down times + Given the study path is "short-tests/005 Thermal fleet - Minimum stable power and min up down times" + When I run antares simulator + Then the simulation takes less than 5 seconds + And the simulation succeeds + And the annual system cost is 2.75816e+07 + And in area "AREA", the units of "base" produce between 400 and 900 MWh hourly + And in area "AREA", the units of "semi base" produce between 100 and 300 MWh hourly + And in area "AREA", the units of "peak" produce between 10 and 100 MWh hourly + # Ideally, we would also check min up & down durations in this test. But is not possible, since clusters defined + # in this test have a unitcount > 1 + # TODO : create similar tests with unitcount = 1, and implement the following steps: + # And in area "AREA", unit "base" respects a minimum up duration of 24 hours, and a minimum down duration of 24 hours + # And in area "AREA", unit "semi base" respects a minimum up duration of 6 hours, and a minimum down duration of 12 hours + # And in area "AREA", unit "peak" respects a minimum up duration of 2 hours, and a minimum down duration of 2 hours + + @fast @short + Scenario: 006 Thermal fleet - Extra costs + # Like previous test, but with extra non-proportional (NP) costs + # NP costs = 1756400 ; OP costs = 2.75816e+07 (like test 5) => Total cost = 2.9338e+07 + Given the study path is "short-tests/006 Thermal fleet - Extra costs" + When I run antares simulator + Then the simulation takes less than 5 seconds + And the simulation succeeds + And the annual system cost is 2.9338e+07 + And in area "AREA", during year 1, total non-proportional cost is 1756400 + And in area "AREA", the units of "base" produce between 400 and 900 MWh hourly + And in area "AREA", the units of "semi base" produce between 100 and 300 MWh hourly + And in area "AREA", the units of "peak" produce between 10 and 100 MWh hourly + # Ideally, we would also check min up & down durations in this test. But is not possible, since clusters defined + # in this test have a unitcount > 1 + # TODO : create similar tests with unitcount = 1, and implement the following steps: + # And in area "AREA", unit "base" respects a minimum up duration of 24 hours, and a minimum down duration of 24 hours + # And in area "AREA", unit "semi base" respects a minimum up duration of 6 hours, and a minimum down duration of 12 hours + # And in area "AREA", unit "peak" respects a minimum up duration of 2 hours, and a minimum down duration of 2 hours + + @fast @short + Scenario: 007 Thermal fleet - Fast unit commitment + # This example is the first of a set of two that are comparing the two unit-commitment modes of Antares. + # Fast mode + # => overall cost is not great, there are a lot of startups, and min up & down time are considered equal + Given the study path is "short-tests/007 Thermal fleet - Fast unit commitment" + When I run antares simulator + Then the simulation takes less than 5 seconds + And the simulation succeeds + And the annual system cost is 2.98912e+07 + And in area "AREA", during year 1, total non-proportional cost is 1861400 + And in area "AREA", the units of "base" produce between 400 and 900 MWh hourly + And in area "AREA", the units of "semi base" produce between 100 and 300 MWh hourly + And in area "AREA", the units of "peak" produce between 10 and 100 MWh hourly + And in area "AREA", during year 1, loss of load lasts 1 hours + And in area "AREA", unsupplied energy on "2 JAN 09:00" of year 1 is of 52 MW + # Ideally, we would also check min up & down durations in this test. But is not possible, since clusters defined + # in this test have a unitcount > 1 + # TODO : create similar tests with unitcount = 1, and implement the following steps: + # And in area "AREA", unit "base" respects a minimum up duration of 24 hours, and a minimum down duration of 24 hours + # And in area "AREA", unit "semi base" respects a minimum up duration of 12 hours, and a minimum down duration of 12 hours + # And in area "AREA", unit "peak" respects a minimum up duration of 2 hours, and a minimum down duration of 2 hours + + @fast @short + Scenario: 008 Thermal fleet - Accurate unit commitment + # Like previous test, but with unit commitment + # => overall cost is better, there are less startups, and min up & down time are not equal + Given the study path is "short-tests/008 Thermal fleet - Accurate unit commitment" + When I run antares simulator + Then the simulation takes less than 5 seconds + And the simulation succeeds + And the annual system cost is 2.97339e+07 + And in area "AREA", during year 1, total non-proportional cost is 1680900 + And in area "AREA", the units of "base" produce between 400 and 900 MWh hourly + And in area "AREA", the units of "semi base" produce between 100 and 300 MWh hourly + And in area "AREA", the units of "peak" produce between 10 and 100 MWh hourly And in area "AREA", during year 1, loss of load lasts 1 hours - And in area "AREA", unsupplied energy on "02 JAN 09:00" of year 1 is of 52 MW + And in area "AREA", unsupplied energy on "2 JAN 09:00" of year 1 is of 52 MW + # Ideally, we would also check min up & down durations in this test. But is not possible, since clusters defined + # in this test have a unitcount > 1 + # TODO : create similar tests with unitcount = 1, and implement the following steps: + # And in area "AREA", unit "base" respects a minimum up duration of 24 hours, and a minimum down duration of 24 hours + # And in area "AREA", unit "semi base" respects a minimum up duration of 6 hours, and a minimum down duration of 12 hours + # And in area "AREA", unit "peak" respects a minimum up duration of 2 hours, and a minimum down duration of 2 hours @fast @short Scenario: 021 Four areas - DC law diff --git a/src/tests/cucumber/features/steps/assertions.py b/src/tests/cucumber/features/steps/assertions.py index dda7d0232c..01ddb6ed99 100644 --- a/src/tests/cucumber/features/steps/assertions.py +++ b/src/tests/cucumber/features/steps/assertions.py @@ -1,4 +1,5 @@ # Custom assertions def assert_double_close(expected, actual, relative_tolerance): - assert abs((actual - expected) / max(1e-6, expected)) <= relative_tolerance \ No newline at end of file + assert abs((actual - expected) / max(1e-6, expected)) <= relative_tolerance, \ + f"Values are not close: expected = {expected} ; actual = {actual}" \ No newline at end of file diff --git a/src/tests/cucumber/features/steps/context_utils.py b/src/tests/cucumber/features/steps/context_utils.py deleted file mode 100644 index 5a7af9dfe3..0000000000 --- a/src/tests/cucumber/features/steps/context_utils.py +++ /dev/null @@ -1,22 +0,0 @@ -# Manage cached output data in "context" object - -from output_utils import * - -def get_annual_system_cost(context): - if context.annual_system_cost is None: - context.annual_system_cost = parse_annual_system_cost(context.output_path) - return context.annual_system_cost - -def get_hourly_values_for_specific_hour(context, area : str, year : int, date : str): - df = get_hourly_values(context, area, year) - day, month, hour = date.split(" ") - return df.loc[(df['Unnamed: 2'] == int(day)) & (df['Unnamed: 3'] == month) & (df['Unnamed: 4'] == hour)] - -def get_hourly_values(context, area : str, year : int): - if context.hourly_values is None: - context.hourly_values = {} - if area not in context.hourly_values: - context.hourly_values[area] = {} - if year not in context.hourly_values[area]: - context.hourly_values[area][year] = parse_hourly_values(context.output_path, area, year) - return context.hourly_values[area][year] \ No newline at end of file diff --git a/src/tests/cucumber/features/steps/output_utils.py b/src/tests/cucumber/features/steps/output_utils.py deleted file mode 100644 index 778a1d36b9..0000000000 --- a/src/tests/cucumber/features/steps/output_utils.py +++ /dev/null @@ -1,37 +0,0 @@ -# Antares outputs parsing - -import os -import pandas -import configparser - -def parse_output_folder_from_logs(logs: bytes) -> str: - for line in logs.splitlines(): - if b'Output folder : ' in line: - return line.split(b'Output folder : ')[1].decode('ascii') - raise LookupError("Could not parse output folder in output logs") - - -def parse_annual_system_cost(output_path: str) -> dict: - file = open(os.path.join(output_path, "annualSystemCost.txt"), 'r') - keys = ["EXP", "STD", "MIN", "MAX"] - annual_system_cost = {} - for line in file.readlines(): - for key in keys: - if key in line: - annual_system_cost[key] = float(line.split(key + " : ")[1]) - return annual_system_cost - - -def parse_simu_time(output_path: str) -> float: - execution_info = configparser.ConfigParser() - execution_info.read(os.path.join(output_path, "execution_info.ini")) - return float(execution_info['durations_ms']['total']) / 1000 - - -def parse_hourly_values(output_path: str, area: str, year: int): - return read_csv(os.path.join(output_path, "economy", "mc-ind", f"{year:05d}", "areas", area, "values-hourly.txt")) - - -def read_csv(file_name): - ignore_rows = [0, 1, 2, 3, 5, 6] - return pandas.read_csv(file_name, skiprows=ignore_rows, sep='\t', low_memory=False) \ No newline at end of file diff --git a/src/tests/cucumber/features/steps/simulator_utils.py b/src/tests/cucumber/features/steps/simulator_utils.py index 91da69a5ad..68f48df180 100644 --- a/src/tests/cucumber/features/steps/simulator_utils.py +++ b/src/tests/cucumber/features/steps/simulator_utils.py @@ -1,35 +1,37 @@ # Methods to run Antares simulator import subprocess -import glob import yaml from pathlib import Path from study_input_handler import study_input_handler -from output_utils import parse_output_folder_from_logs +from study_output_handler import study_output_handler def get_solver_path(): with open("conf.yaml") as file: - content = yaml.full_load(file) + content = yaml.full_load(file) return content.get("antares-solver") -SOLVER_PATH = get_solver_path() # we only need to run this once + +SOLVER_PATH = get_solver_path() # we only need to run this once def run_simulation(context): - activate_simu_outputs(context) # TODO : remove this and update studies instead + init_simu(context) command = build_antares_solver_command(context) print(f"Running command: {command}") process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) out, err = process.communicate() context.output_path = parse_output_folder_from_logs(out) context.return_code = process.returncode - context.annual_system_cost = None - context.hourly_values = None + context.soh = study_output_handler(context.output_path) -def activate_simu_outputs(context): +def init_simu(context): sih = study_input_handler(Path(context.study_path)) + # read metadata + context.nbyears = int(sih.get_value(variable="nbyears", file_nick_name="general")) + # activate year-by-year results # TODO : remove this and update studies instead sih.set_value(variable="synthesis", value="true", file_nick_name="general") sih.set_value(variable="year-by-year", value="true", file_nick_name="general") @@ -45,3 +47,9 @@ def build_antares_solver_command(context): command.append('--force-parallel=4') return command + +def parse_output_folder_from_logs(logs: bytes) -> str: + for line in logs.splitlines(): + if b'Output folder : ' in line: + return line.split(b'Output folder : ')[1].decode('ascii') + raise LookupError("Could not parse output folder in output logs") diff --git a/src/tests/cucumber/features/steps/steps.py b/src/tests/cucumber/features/steps/steps.py index f49bed8868..ef9ede4c2b 100644 --- a/src/tests/cucumber/features/steps/steps.py +++ b/src/tests/cucumber/features/steps/steps.py @@ -1,14 +1,18 @@ # Gherkins test steps definitions import os +import pathlib + from behave import * -from simulator_utils import * + from assertions import * -from context_utils import * +from simulator_utils import * + @given('the study path is "{string}"') def study_path_is(context, string): - context.study_path = os.path.join("..", "resources", "Antares_Simulator_Tests_NR" , string.replace("/", os.sep)) + context.study_path = os.path.join("..", "resources", "Antares_Simulator_Tests_NR", string.replace("/", os.sep)) + @when('I run antares simulator') def run_antares(context): @@ -18,50 +22,104 @@ def run_antares(context): context.parallel = False run_simulation(context) + +def after_feature(context, feature): + # post-processing a test: clean up output files to avoid taking up all the disk space + if (context.output_path != None): + pathlib.Path.rmdir(context.output_path) + + @then('the simulation succeeds') def simu_success(context): assert context.return_code == 0 + @then('the simulation fails') def simu_success(context): assert context.return_code != 0 -@then('the expected value of the annual system cost is {value}') + +@then('the expected value of the annual system cost is {value:g}') def check_annual_cost_expected(context, value): - assert_double_close(float(value), get_annual_system_cost(context)["EXP"], 0.001) + assert_double_close(value, context.soh.get_annual_system_cost()["EXP"], 0.001) -@then('the minimum annual system cost is {value}') + +@then('the minimum annual system cost is {value:g}') def check_annual_cost_min(context, value): - assert_double_close(float(value), get_annual_system_cost(context)["MIN"], 0.001) + assert_double_close(value, context.soh.get_annual_system_cost()["MIN"], 0.001) + -@then('the maximum annual system cost is {value}') +@then('the maximum annual system cost is {value:g}') def check_annual_cost_max(context, value): - assert_double_close(float(value), get_annual_system_cost(context)["MAX"], 0.001) + assert_double_close(value, context.soh.get_annual_system_cost()["MAX"], 0.001) + @then('the annual system cost is') def check_annual_cost(context): for row in context.table: - assert_double_close(float(row["EXP"]), get_annual_system_cost(context)["EXP"], 0.001) - assert_double_close(float(row["STD"]), get_annual_system_cost(context)["STD"], 0.001) - assert_double_close(float(row["MIN"]), get_annual_system_cost(context)["MIN"], 0.001) - assert_double_close(float(row["MAX"]), get_annual_system_cost(context)["MAX"], 0.001) + assert_double_close(float(row["EXP"]), context.soh.get_annual_system_cost()["EXP"], 0.001) + assert_double_close(float(row["STD"]), context.soh.get_annual_system_cost()["STD"], 0.001) + assert_double_close(float(row["MIN"]), context.soh.get_annual_system_cost()["MIN"], 0.001) + assert_double_close(float(row["MAX"]), context.soh.get_annual_system_cost()["MAX"], 0.001) + + +@then('the annual system cost is {one_year_value:g}') +def check_annual_cost(context, one_year_value): + assert_double_close(one_year_value, context.soh.get_annual_system_cost()["EXP"], 0.001) + assert_double_close(0, context.soh.get_annual_system_cost()["STD"], 0.001) + assert_double_close(one_year_value, context.soh.get_annual_system_cost()["MIN"], 0.001) + assert_double_close(one_year_value, context.soh.get_annual_system_cost()["MAX"], 0.001) + -@then('the simulation takes less than {seconds} seconds') +@then('the simulation takes less than {seconds:g} seconds') def check_simu_time(context, seconds): - actual_simu_time = parse_simu_time(context.output_path) - assert actual_simu_time <= float(seconds) + assert context.soh.get_simu_time() <= seconds -@then('in area "{area}", during year {year}, loss of load lasts {lold_hours} hours') + +@then('in area "{area}", during year {year:d}, loss of load lasts {lold_hours:d} hours') def check_lold_duration(context, area, year, lold_hours): - assert int(lold_hours) == get_hourly_values(context, area.lower(), int(year))["LOLD"].sum() + assert lold_hours == context.soh.get_loss_of_load_duration_h(area, year) + -@then('in area "{area}", unsupplied energy on "{date}" of year {year} is of {lold_value_mw} MW') +@then('in area "{area}", unsupplied energy on "{date}" of year {year:d} is of {lold_value_mw:g} MW') def check_lold_value(context, area, date, year, lold_value_mw): - actual_unsp_energ = get_hourly_values_for_specific_hour(context, area.lower(), int(year), date)["UNSP. ENRG"].sum() - assert_double_close(float(lold_value_mw), actual_unsp_energ, 0.001) + actual_unsp_energ = context.soh.get_unsupplied_energy_mwh(area, year, date) + assert_double_close(lold_value_mw, actual_unsp_energ, 0.001) + + +@then( + 'in area "{area}", during year {year:d}, hourly production of "{prod_name}" is always {comparator_and_hourly_prod} MWh') +def check_prod_for_specific_year(context, area, year, prod_name, comparator_and_hourly_prod): + expected_prod = float(comparator_and_hourly_prod.split(" ")[-1]) + actual_hourly_prod = context.soh.get_hourly_prod_mwh(area, year, prod_name) + if "greater than" in comparator_and_hourly_prod: + ok = actual_hourly_prod >= expected_prod + elif "equal to" in comparator_and_hourly_prod: + ok = actual_hourly_prod - expected_prod <= 1e-6 + else: + raise NotImplementedError(f"Unknown comparator '{comparator_and_hourly_prod}'") + if "zero or" in comparator_and_hourly_prod: + ok = ok | (actual_hourly_prod == 0) + assert ok.all() + + +@then('in area "{area}", hourly production of "{prod_name}" is always {comparator_and_hourly_prod} MWh') +def check_prod_for_all_years(context, area, prod_name, comparator_and_hourly_prod): + for year in range(1, context.nbyears + 1): + check_prod_for_specific_year(context, area, year, prod_name, comparator_and_hourly_prod) + + +@step('in area "{area}", during year {year:d}, total non-proportional cost is {np_cost:g}') +def check_np_cost_for_specific_year(context, area, year, np_cost): + assert_double_close(np_cost, context.soh.get_non_proportional_cost(area, year), 1e-6) -def after_feature(context, feature): - # post-processing a test: clean up output files to avoid taking up all the disk space - if (context.output_path != None): - pathlib.Path.rmdir(context.output_path) +@then('in area "{area}", the units of "{prod_name}" produce between {min_p:g} and {max_p:g} MWh hourly') +def check_pmin_pmax(context, area, prod_name, min_p, max_p): + for year in range(1, context.nbyears + 1): + actual_hourly_prod = context.soh.get_hourly_prod_mwh(area, year, prod_name) + actual_n_dispatched_units = context.soh.get_hourly_n_dispatched_units(area, year, prod_name) + assert (actual_hourly_prod <= actual_n_dispatched_units.apply( + lambda n: n * max_p)).all(), f"max_p constraint not respected during year {year}" + assert (actual_hourly_prod >= actual_n_dispatched_units.apply( + lambda n: n * min_p)).all(), f"min_p constraint not respected during year {year}" diff --git a/src/tests/cucumber/features/steps/study_input_handler.py b/src/tests/cucumber/features/steps/study_input_handler.py index 90f27d7221..b6f83bfe04 100644 --- a/src/tests/cucumber/features/steps/study_input_handler.py +++ b/src/tests/cucumber/features/steps/study_input_handler.py @@ -1,6 +1,3 @@ -# Currently used to activate simulation outputs -# TODO : remove this and update parameters in simulation input files - import os @@ -12,6 +9,35 @@ def __init__(self, study_root_directory): self.files_path["desktop"] = self.study_root_dir / "Desktop.ini" self.files_path["general"] = self.study_root_dir / "settings" / "generaldata.ini" self.files_path["study"] = self.study_root_dir / "study.antares" + self.files_path["thermal"] = self.study_root_dir / "study.antares" + + def get_value(self, variable, file_nick_name): + # File path + file = self.files_path[file_nick_name] + + # Reading the file content (content in) + with open(file) as f: + # Searching variable and setting its value in a tmp content + for line in f: + if line.strip().startswith(variable): + return line.split('=')[1].strip() + + def get_input(self, input_file, section, variable): + # File path + file = self.study_root_dir / "input" / input_file.replace("/", os.sep) + correct_section = False + # Reading the file content (content in) + with open(file) as f: + # Searching variable and setting its value in a tmp content + for line in f: + if line.startswith("["): + if f"[{section}]" in line: + correct_section = True + else: + correct_section = False + else: + if correct_section and line.strip().startswith(variable): + return line.split('=')[1].strip() def set_value(self, variable, value, file_nick_name): # File path diff --git a/src/tests/cucumber/features/steps/study_output_handler.py b/src/tests/cucumber/features/steps/study_output_handler.py new file mode 100644 index 0000000000..1c20e5bf08 --- /dev/null +++ b/src/tests/cucumber/features/steps/study_output_handler.py @@ -0,0 +1,84 @@ +# Antares outputs parsing + +import os +import pandas as pd +import configparser +from enum import Enum + + +class result_type(Enum): + VALUES = "values" + DETAILS = "details" + + +class study_output_handler: + + def __init__(self, study_output_path): + self.study_output_path = study_output_path + self.annual_system_cost = None + self.hourly_results = {result_type.VALUES: None, result_type.DETAILS: None} + + def get_annual_system_cost(self): + if self.annual_system_cost is None: + self.__parse_annual_system_cost() + return self.annual_system_cost + + def __parse_annual_system_cost(self): + file = open(os.path.join(self.study_output_path, "annualSystemCost.txt"), 'r') + keys = ["EXP", "STD", "MIN", "MAX"] + annual_system_cost = {} + for line in file.readlines(): + for key in keys: + if key in line: + annual_system_cost[key] = float(line.split(key + " : ")[1]) + self.annual_system_cost = annual_system_cost + + def get_simu_time(self) -> float: + execution_info = configparser.ConfigParser() + execution_info.read(os.path.join(self.study_output_path, "execution_info.ini")) + return float(execution_info['durations_ms']['total']) / 1000 + + def __read_csv(self, file_name) -> pd.DataFrame: + ignore_rows = [0, 1, 2, 3, 6] + absolute_path = os.path.join(self.study_output_path, file_name.replace("/", os.sep)) + return pd.read_csv(absolute_path, header=[0, 1], skiprows=ignore_rows, sep='\t', low_memory=False) + + def __if_none_then_parse(self, rs: result_type, area, year, file_name: str): + if self.hourly_results[rs] is None: + self.hourly_results[rs] = {} + if area not in self.hourly_results[rs]: + self.hourly_results[rs][area] = {} + if year not in self.hourly_results[rs][area]: + # parse file + self.hourly_results[rs][area][year] = self.__read_csv( + f"economy/mc-ind/{year:05d}/areas/{area}/{file_name}") + # add datetime column by concatenating unnamed columns 2 (day), 3 (month), 4 (hour) + cols = ['Unnamed: 2_level_0', 'Unnamed: 3_level_0', 'Unnamed: 4_level_0'] + self.hourly_results[rs][area][year]["datetime"] = self.hourly_results[rs][area][year][cols].apply( + lambda row: ' '.join(row.values.astype(str)), axis=1) + return self.hourly_results[rs][area][year] + + def __get_values_hourly(self, area: str, year: int): + return self.__if_none_then_parse(result_type.VALUES, area.lower(), year, "values-hourly.txt") + + def __get_values_hourly_for_specific_hour(self, area: str, year: int, datetime: str): + df = self.__get_values_hourly(area, year) + return df.loc[df['datetime'] == datetime] + + def __get_details_hourly(self, area: str, year: int): + return self.__if_none_then_parse(result_type.DETAILS, area.lower(), year, "details-hourly.txt") + + def get_hourly_prod_mwh(self, area: str, year: int, prod_name: str) -> pd.Series: + return self.__get_details_hourly(area, year)[prod_name]['MWh'] + + def get_hourly_n_dispatched_units(self, area: str, year: int, prod_name: str) -> pd.Series: + return self.__get_details_hourly(area, year)[prod_name]['NODU'] + + def get_loss_of_load_duration_h(self, area: str, year: int) -> int: + return self.__get_values_hourly(area, year)["LOLD"]["Hours"].sum() + + def get_unsupplied_energy_mwh(self, area: str, year: int, date: str) -> float: + return self.__get_values_hourly_for_specific_hour(area, year, date)["UNSP. ENRG"]["MWh"].sum() + + def get_non_proportional_cost(self, area: str, year: int) -> float: + return self.__get_values_hourly(area, year)["NP COST"]["Euro"].sum() diff --git a/src/tests/cucumber/requirements.txt b/src/tests/cucumber/requirements.txt index fd5e363e68..2e68e9859a 100644 --- a/src/tests/cucumber/requirements.txt +++ b/src/tests/cucumber/requirements.txt @@ -1,2 +1,3 @@ behave -pyyaml \ No newline at end of file +pyyaml +pandas \ No newline at end of file