Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(outputs): build outputs tree based on filesystem #2064

Merged
merged 20 commits into from
Jun 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
60de5cd
feat(filesystem): start work on mc-all and mc-ind
MartinBelthle Jun 11, 2024
6a12eb8
refactor(filesystem): simplify code
MartinBelthle Jun 11, 2024
2448532
refactor(filesystem): improve the code
MartinBelthle Jun 12, 2024
042e663
refactor(filesystem): fix mypy issue
MartinBelthle Jun 12, 2024
e123765
refactor(filesystem): remove todos
MartinBelthle Jun 12, 2024
033b23b
fix(filesystem): use stem instead of name to build files
MartinBelthle Jun 12, 2024
ce5f5bb
fix(tests): adapt tests to new behavior
MartinBelthle Jun 12, 2024
110936e
refactor(filesystem): simplify code
MartinBelthle Jun 12, 2024
4718e2c
feat(filesystem): apply the same changes to ts-numbers and ts-generator
MartinBelthle Jun 12, 2024
e15afd3
fix(filesystem): do not build inputs based on filesystem
MartinBelthle Jun 12, 2024
9478573
fix(filesystem): change path
MartinBelthle Jun 12, 2024
32a0fc1
refactor(filesystem): remove unused initialization
MartinBelthle Jun 12, 2024
1b66404
refactor(filesystem): remove useless file
MartinBelthle Jun 12, 2024
5f26e73
refactor(filesystem): resolve some comments
MartinBelthle Jun 18, 2024
875a515
refactor(filesystem): resolve other comments
MartinBelthle Jun 18, 2024
72600ce
refactor(filesystem): resolve one comment
MartinBelthle Jun 18, 2024
21446da
feat(tests): add UT for sets
MartinBelthle Jun 18, 2024
c725ac1
feat(tests): add UT for binding constraint ts numbers
MartinBelthle Jun 18, 2024
9a7e67b
docs(filesystem): add little doc
MartinBelthle Jun 18, 2024
f11baf7
fix(tests): change output count
MartinBelthle Jun 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
from antarest.study.storage.rawstudy.model.filesystem.inode import TREE, INode
from antarest.study.storage.rawstudy.model.filesystem.matrix.input_series_matrix import InputSeriesMatrix

TXT_PATTERN = "*.txt"


class AreaMatrixList(FolderNode):
"""
Expand Down Expand Up @@ -60,14 +62,17 @@ def build(self) -> TREE:
A dictionary of child nodes, where the key is the matrix file name
and the value is the corresponding :class:`InputSeriesMatrix` node.
"""
children: TREE = {
f"{self.prefix}{area}": self.matrix_class(
self.context,
self.config.next_file(f"{self.prefix}{area}.txt"),
**self.additional_matrix_params,
children: TREE = {}
if self.prefix: # Corresponds to the inputs
files = self.config.area_names()
else: # Corresponds to the outputs
files = [d.with_suffix("").name for d in self.config.path.iterdir()]

for file in files:
name = f"{self.prefix}{file}"
children[name] = self.matrix_class(
self.context, self.config.next_file(f"{name}.txt"), **self.additional_matrix_params
)
for area in self.config.area_names()
}
return children


Expand Down Expand Up @@ -105,7 +110,7 @@ def build(self) -> TREE:
"""Builds the folder structure and creates child nodes representing each matrix file."""
return {
file.stem: self.matrix_class(self.context, self.config.next_file(file.name))
for file in self.config.path.glob("*.txt")
for file in self.config.path.glob(TXT_PATTERN)
}


Expand All @@ -124,15 +129,28 @@ def __init__(
def build(self) -> TREE:
# Note that cluster IDs are case-insensitive, but series IDs are in lower case.
# For instance, if your cluster ID is "Base", then the series ID will be "base".
series_ids = map(str.lower, self.config.get_thermal_ids(self.area))
children: TREE = {
series_id: self.matrix_class(self.context, self.config.next_file(f"{series_id}.txt"))
for series_id in series_ids
series_files = self.config.path.glob(TXT_PATTERN)
return {
series.stem: self.matrix_class(self.context, self.config.next_file(series.name)) for series in series_files
}
return children


class AreaMultipleMatrixList(FolderNode):
MartinBelthle marked this conversation as resolved.
Show resolved Hide resolved
"""
Node representing a folder structure containing multiple matrix files for each area.

Example of tree structure:

.. code-block:: text

ts-numbers/thermal
├── at
│ ├── cluster_gas.txt
│ └── cluster2_gas.txt
└── be
└── cluster_nuclear.txt
"""

def __init__(
self,
context: ContextServer,
Expand All @@ -156,13 +174,14 @@ def __init__(
self.matrix_class = matrix_class

def build(self) -> TREE:
folders = [d.name for d in self.config.path.iterdir() if d.is_dir()]
children: TREE = {
area: self.klass(
self.context,
self.config.next_file(area),
area,
self.matrix_class,
)
for area in self.config.area_names()
for area in folders
}
return children
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
from antarest.study.storage.rawstudy.model.filesystem.common.area_matrix_list import AreaMatrixList
from antarest.study.storage.rawstudy.model.filesystem.common.prepro import InputPrepro
from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig
from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer
from antarest.study.storage.rawstudy.model.filesystem.folder_node import FolderNode
from antarest.study.storage.rawstudy.model.filesystem.inode import TREE
from antarest.study.storage.rawstudy.model.filesystem.matrix.constants import default_scenario_hourly


class InputPreproSeries(FolderNode):
def __init__(self, context: ContextServer, config: FileStudyTreeConfig, prefix: str):
"""
Represents a folder structure, which contains a "prepro" and a time series structure.

Example of tree structure:

.. code-block:: text

input/load/
├── prepro
│ ├── correlation.ini
│ ├── store_in
│ │ ├── conversion.txt
│ │ ├── data.txt
│ │ ├── k.txt
│ │ ├── settings.ini
│ │ └── translation.txt
│ └── store_out
│ ├── conversion.txt
│ ├── data.txt
│ ├── k.txt
│ ├── settings.ini
│ └── translation.txt
└── series
├── load_store_in.txt
└── load_store_out.txt
"""

super().__init__(context, config)
self.prefix = prefix

def build(self) -> TREE:
children: TREE = {
"prepro": InputPrepro(self.context, self.config.next_file("prepro")),
"series": AreaMatrixList(
self.context,
self.config.next_file("series"),
prefix=self.prefix,
additional_matrix_params={"default_empty": default_scenario_hourly},
),
}
return children
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,14 @@
from antarest.study.storage.rawstudy.model.filesystem.root.input.bindingconstraints.bindingcontraints import (
BindingConstraints,
)
from antarest.study.storage.rawstudy.model.filesystem.root.input.commons.prepro_series import InputPreproSeries
from antarest.study.storage.rawstudy.model.filesystem.root.input.hydro.hydro import InputHydro
from antarest.study.storage.rawstudy.model.filesystem.root.input.link.link import InputLink
from antarest.study.storage.rawstudy.model.filesystem.root.input.load.load import InputLoad
from antarest.study.storage.rawstudy.model.filesystem.root.input.miscgen.miscgen import InputMiscGen
from antarest.study.storage.rawstudy.model.filesystem.root.input.renewables.renewable import ClusteredRenewables
from antarest.study.storage.rawstudy.model.filesystem.root.input.reserves.reserves import InputReserves
from antarest.study.storage.rawstudy.model.filesystem.root.input.solar.solar import InputSolar
from antarest.study.storage.rawstudy.model.filesystem.root.input.st_storage.st_storage import InputSTStorage
from antarest.study.storage.rawstudy.model.filesystem.root.input.thermal.thermal import InputThermal
from antarest.study.storage.rawstudy.model.filesystem.root.input.wind.wind import InputWind


class Input(FolderNode):
Expand All @@ -31,12 +29,12 @@ def build(self) -> TREE:
"bindingconstraints": BindingConstraints(self.context, config.next_file("bindingconstraints")),
"hydro": InputHydro(self.context, config.next_file("hydro")),
"links": InputLink(self.context, config.next_file("links")),
"load": InputLoad(self.context, config.next_file("load")),
"load": InputPreproSeries(self.context, config.next_file("load"), "load_"),
"misc-gen": InputMiscGen(self.context, config.next_file("misc-gen")),
"reserves": InputReserves(self.context, config.next_file("reserves")),
"solar": InputSolar(self.context, config.next_file("solar")),
"solar": InputPreproSeries(self.context, config.next_file("solar"), "solar_"),
"thermal": InputThermal(self.context, config.next_file("thermal")),
"wind": InputWind(self.context, config.next_file("wind")),
"wind": InputPreproSeries(self.context, config.next_file("wind"), "wind_"),
}

has_renewables = config.version >= 810 and EnrModelling(config.enr_modelling) == EnrModelling.CLUSTERS
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ def __init__(
config: FileStudyTreeConfig,
area: str,
):
FolderNode.__init__(self, context, config)
super().__init__(context, config)
self.area = area

def build(self) -> TREE:
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def __init__(
config: FileStudyTreeConfig,
area: str,
):
FolderNode.__init__(self, context, config)
super().__init__(context, config)
self.area = area

def build(self) -> TREE:
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def __init__(
config: FileStudyTreeConfig,
area: str,
):
FolderNode.__init__(self, context, config)
super().__init__(context, config)
self.area = area

def build(self) -> TREE:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def __init__(
config: FileStudyTreeConfig,
area: str,
):
FolderNode.__init__(self, context, config)
super().__init__(context, config)
self.area = area

def build(self) -> TREE:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def __init__(
config: FileStudyTreeConfig,
area: str,
):
FolderNode.__init__(self, context, config)
super().__init__(context, config)
self.area = area

def build(self) -> TREE:
Expand Down

This file was deleted.

Loading
Loading