Skip to content

Commit

Permalink
revamp NEMS configuration to not have duplicate model objects
Browse files Browse the repository at this point in the history
  • Loading branch information
@zacharyburnett
zacharyburnett committed Jul 23, 2021
1 parent e40f43f commit e153bdc
Show file tree
Hide file tree
Showing 36 changed files with 83 additions and 196 deletions.
145 changes: 33 additions & 112 deletions coupledmodeldriver/configure/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,13 @@
from typing import Any, Union

from adcircpy.server import SlurmConfig
import nemspy
from nemspy import ModelingSystem
from nemspy.model.base import ModelEntry, ModelMeshEntry
from nemspy.model.base import ModelEntry

from coupledmodeldriver.platforms import Platform
from coupledmodeldriver.script import SlurmEmailType
from coupledmodeldriver.utilities import convert_to_json, convert_value, LOGGER

NEMS_MODEL_ENTRIES = [
*(cls for name, cls in nemspy.model.__dict__.items() if isinstance(cls, type)),
ModelEntry,
]


class ConfigurationJSON(ABC):
name: str
Expand Down Expand Up @@ -67,7 +61,7 @@ def move_paths(self, relative: PathLike):

for name, value in self.configuration.items():
if isinstance(value, Path) and not value.is_absolute():
self[name] = PurePosixPath(move_path(value, relative))
self[name] = PurePosixPath(move_path(value, relative).resolve())

def relative_to(self, path: PathLike, inplace: bool = False) -> 'ConfigurationJSON':
instance = copy(self) if not inplace else self
Expand Down Expand Up @@ -371,6 +365,32 @@ def __init__(self, attributes: {str: Any} = None, **kwargs):
self['attributes'] = attributes


class NEMSCapJSON(ConfigurationJSON, ABC):
default_processors: int
field_types = {
'processors': int,
'nems_parameters': {str: str},
}

def __init__(self, processors: int = None, nems_parameters: {str: str} = None, **kwargs):
if processors is None:
processors = self.default_processors
if nems_parameters is None:
nems_parameters = {}
if 'fields' not in kwargs:
kwargs['fields'] = {}
kwargs['fields'].update(NEMSCapJSON.field_types)

ConfigurationJSON.__init__(self, **kwargs)

self['processors'] = processors
self['nems_parameters'] = nems_parameters

@abstractmethod
def nemspy_entry(self) -> ModelEntry:
raise NotImplementedError()


class NEMSJSON(ConfigurationJSON):
name = 'NEMS'
default_filename = f'configure_nems.json'
Expand All @@ -379,7 +399,6 @@ class NEMSJSON(ConfigurationJSON):
'modeled_start_time': datetime,
'modeled_end_time': datetime,
'interval': timedelta,
'models': [ModelEntry],
'connections': [[str]],
'mediations': [str],
'sequence': [str],
Expand All @@ -391,7 +410,6 @@ def __init__(
modeled_start_time: datetime,
modeled_end_time: datetime,
interval: timedelta = None,
models: [ModelEntry] = None,
connections: [[str]] = None,
mediations: [[str]] = None,
sequence: [str] = None,
Expand All @@ -407,18 +425,19 @@ def __init__(
self['modeled_start_time'] = modeled_start_time
self['modeled_end_time'] = modeled_end_time
self['interval'] = interval
self['models'] = models
self['connections'] = connections
self['mediations'] = mediations
self['sequence'] = sequence

@property
def nemspy_modeling_system(self) -> ModelingSystem:
def to_nemspy(self, models: [NEMSCapJSON]) -> ModelingSystem:
models = [
model.nemspy_entry if isinstance(model, NEMSCapJSON) else model for model in models
]
modeling_system = ModelingSystem(
start_time=self['modeled_start_time'],
end_time=self['modeled_end_time'],
interval=self['interval'],
**{model.model_type.value.lower(): model for model in self['models']},
**{model.model_type.value.lower(): model for model in models},
)
for connection in self['connections']:
modeling_system.connect(*connection)
Expand All @@ -432,22 +451,6 @@ def nemspy_modeling_system(self) -> ModelingSystem:

return modeling_system

def move_paths(self, relative: PathLike):
super().move_paths(relative)
for model in self['models']:
if isinstance(model, ModelMeshEntry):
model.filename = move_path(model.filename, relative)

def relative_to(self, path: PathLike, inplace: bool = False) -> 'NEMSJSON':
instance = super().relative_to(path, inplace)
for model in instance['models']:
if isinstance(model, ModelMeshEntry):
model.filename = Path(os.path.relpath(model.filename, path))
return instance

def to_nemspy(self) -> ModelingSystem:
return self.nemspy_modeling_system

@classmethod
def from_nemspy(cls, modeling_system: ModelingSystem, executable_path: PathLike = None):
if executable_path is None:
Expand All @@ -462,88 +465,6 @@ def from_nemspy(cls, modeling_system: ModelingSystem, executable_path: PathLike
sequence=modeling_system.sequence,
)

@classmethod
def from_string(cls, string: str) -> 'NEMSJSON':
configuration = json.loads(string)

if 'models' in configuration:
if configuration['models'] is not None:
for index, value in enumerate(configuration['models']):
for model_entry_type in NEMS_MODEL_ENTRIES:
try:
value = model_entry_type.from_string(value)
configuration['models'][index] = value
break
except:
pass

configuration = {
key.lower(): convert_value(value, cls.field_types[key])
if key in cls.field_types
else convert_to_json(value)
for key, value in configuration.items()
}

return cls.from_dict(configuration)

@classmethod
def from_file(cls, filename: PathLike) -> 'NEMSJSON':
if not isinstance(filename, Path):
filename = Path(filename)

if filename.is_dir():
filename = filename / cls.default_filename

with open(filename) as file:
LOGGER.debug(f'reading file "{os.path.relpath(filename.resolve(), Path.cwd())}"')
configuration = json.load(file)

if 'models' in configuration:
if configuration['models'] is not None:
for index, value in enumerate(configuration['models']):
for model_entry_type in NEMS_MODEL_ENTRIES:
try:
value = model_entry_type.from_string(value)
configuration['models'][index] = value
break
except:
pass

configuration = {
key.lower(): convert_value(value, cls.field_types[key])
if key in cls.field_types
else convert_to_json(value)
for key, value in configuration.items()
}

return cls.from_dict(configuration)


class NEMSCapJSON(ConfigurationJSON, ABC):
default_processors: int
field_types = {
'processors': int,
'nems_parameters': {str: str},
}

def __init__(self, processors: int = None, nems_parameters: {str: str} = None, **kwargs):
if processors is None:
processors = self.default_processors
if nems_parameters is None:
nems_parameters = {}
if 'fields' not in kwargs:
kwargs['fields'] = {}
kwargs['fields'].update(NEMSCapJSON.field_types)

ConfigurationJSON.__init__(self, **kwargs)

self['processors'] = processors
self['nems_parameters'] = nems_parameters

@abstractmethod
def nemspy_entry(self) -> ModelEntry:
raise NotImplementedError()


def move_path(path: PathLike, move: Union[PathLike, int]) -> Path:
if not isinstance(path, Path):
Expand Down
12 changes: 1 addition & 11 deletions coupledmodeldriver/configure/configure.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,7 @@
from pathlib import Path
from typing import Any, Collection, Mapping, Union

from nemspy.model.base import ModelEntry

from coupledmodeldriver.configure.base import ConfigurationJSON, ModelDriverJSON, NEMSCapJSON
from coupledmodeldriver.configure.base import ConfigurationJSON, ModelDriverJSON
from coupledmodeldriver.configure.forcings.base import ADCIRCPY_FORCING_CLASSES, ForcingJSON
from coupledmodeldriver.utilities import LOGGER

Expand Down Expand Up @@ -53,14 +51,6 @@ def configurations(self, configurations: [ConfigurationJSON]):
for name, configuration in configurations.items():
self[name] = configuration

@property
def nemspy_entries(self) -> [ModelEntry]:
return [
configuration.nemspy_entry
for configuration in self.__configurations.values()
if isinstance(configuration, NEMSCapJSON)
]

def move_paths(self, relative: PathLike):
for configuration in self.configurations:
configuration.move_paths(relative)
Expand Down
20 changes: 12 additions & 8 deletions coupledmodeldriver/generate/adcirc/configure.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from adcircpy import AdcircMesh, AdcircRun
from nemspy import ModelingSystem
from nemspy.model.base import ModelEntry

from coupledmodeldriver.configure import NEMSJSON
from coupledmodeldriver.configure.base import (
Expand Down Expand Up @@ -273,7 +274,6 @@ def __init__(
modeled_start_time=modeled_start_time,
modeled_end_time=modeled_end_time,
interval=nems_interval,
models=self.nemspy_entries,
connections=nems_connections,
mediations=nems_mediations,
sequence=nems_sequence,
Expand All @@ -284,17 +284,23 @@ def __init__(
for forcing in forcings:
self.add_forcing(forcing)

self['slurm'].tasks = self['nems'].nemspy_modeling_system.processors
self['slurm'].tasks = self.nemspy_modeling_system.processors

@property
def nemspy_entries(self) -> [ModelEntry]:
return [
configuration.nemspy_entry
for configuration in self.configurations
if isinstance(configuration, NEMSCapJSON)
]

@property
def nemspy_modeling_system(self) -> ModelingSystem:
return self['nems'].nemspy_modeling_system
return self['nems'].to_nemspy(self.nemspy_entries)

def add_forcing(self, forcing: ForcingJSON):
if forcing not in self:
forcing = self[self.add(forcing)]
if isinstance(forcing, NEMSCapJSON):
self['nems']['models'].append(forcing.nemspy_entry)
self['adcirc'].add_forcing(forcing)

def __copy__(self) -> 'NEMSADCIRCRunConfiguration':
Expand Down Expand Up @@ -338,6 +344,4 @@ def read_directory(
supplementary = set()
supplementary.update(NEMSADCIRCRunConfiguration.SUPPLEMENTARY)

instance = super().read_directory(directory, required, supplementary)
instance['nems']['models'] = instance.nemspy_entries
return instance
return super().read_directory(directory, required, supplementary)
10 changes: 6 additions & 4 deletions coupledmodeldriver/generate/adcirc/generate.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,10 +263,11 @@ async def write_spinup_directory(

spinup_adcircpy_driver = spinup_configuration.adcircpy_driver

spinup_configuration.relative_to(spinup_directory, inplace=True)
if relative_paths:
spinup_configuration.relative_to(spinup_directory, inplace=True)

if use_nems:
spinup_nems = spinup_configuration['nems'].nemspy_modeling_system
spinup_nems = spinup_configuration.nemspy_modeling_system
spinup_nems = ModelingSystem(
spinup_nems.start_time - spinup_duration,
spinup_nems.start_time,
Expand Down Expand Up @@ -415,10 +416,11 @@ async def write_run_directory(

run_adcircpy_driver = run_configuration.adcircpy_driver

run_configuration.relative_to(run_directory, inplace=True)
if relative_paths:
run_configuration.relative_to(run_directory, inplace=True)

if use_nems:
run_nems = run_configuration['nems'].nemspy_modeling_system
run_nems = run_configuration.nemspy_modeling_system
run_processors = run_nems.processors
run_model_executable = run_configuration['nems']['executable_path']
else:
Expand Down
2 changes: 1 addition & 1 deletion tests/data/reference/adcirc/hera_shinnecock_ike/runs/unperturbed/fort.15
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
created on 2021-07-08 16:11 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
created on 2021-07-23 12:06 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
Shinacock Inlet Coarse Grid ! RUNID - 24 CHARACTER ALPANUMERIC RUN IDENTIFICATION
1 ! NFOVER - NONFATAL ERROR OVERRIDE OPTION
1 ! NABOUT - ABREVIATED OUTPUT OPTION PARAMETER
Expand Down
2 changes: 1 addition & 1 deletion tests/data/reference/adcirc/hera_shinnecock_ike/spinup/fort.15
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
created on 2021-07-08 16:10 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
created on 2021-07-23 12:05 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
Shinacock Inlet Coarse Grid ! RUNID - 24 CHARACTER ALPANUMERIC RUN IDENTIFICATION
1 ! NFOVER - NONFATAL ERROR OVERRIDE OPTION
1 ! NABOUT - ABREVIATED OUTPUT OPTION PARAMETER
Expand Down
2 changes: 1 addition & 1 deletion tests/data/reference/adcirc/hera_shinnecock_ike_nospinup/runs/unperturbed/fort.15
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
created on 2021-07-08 16:10 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
created on 2021-07-23 12:04 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
Shinacock Inlet Coarse Grid ! RUNID - 24 CHARACTER ALPANUMERIC RUN IDENTIFICATION
1 ! NFOVER - NONFATAL ERROR OVERRIDE OPTION
1 ! NABOUT - ABREVIATED OUTPUT OPTION PARAMETER
Expand Down
2 changes: 1 addition & 1 deletion tests/data/reference/adcirc/local_shinnecock_ike/runs/unperturbed/fort.15
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
created on 2021-07-08 16:11 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
created on 2021-07-23 12:06 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
Shinacock Inlet Coarse Grid ! RUNID - 24 CHARACTER ALPANUMERIC RUN IDENTIFICATION
1 ! NFOVER - NONFATAL ERROR OVERRIDE OPTION
1 ! NABOUT - ABREVIATED OUTPUT OPTION PARAMETER
Expand Down
2 changes: 1 addition & 1 deletion tests/data/reference/adcirc/local_shinnecock_ike/spinup/fort.15
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
created on 2021-07-08 16:10 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
created on 2021-07-23 12:05 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
Shinacock Inlet Coarse Grid ! RUNID - 24 CHARACTER ALPANUMERIC RUN IDENTIFICATION
1 ! NFOVER - NONFATAL ERROR OVERRIDE OPTION
1 ! NABOUT - ABREVIATED OUTPUT OPTION PARAMETER
Expand Down
2 changes: 1 addition & 1 deletion tests/data/reference/adcirc/stampede2_shinnecock_ike/runs/unperturbed/fort.15
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
created on 2021-07-08 16:11 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
created on 2021-07-23 12:06 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
Shinacock Inlet Coarse Grid ! RUNID - 24 CHARACTER ALPANUMERIC RUN IDENTIFICATION
1 ! NFOVER - NONFATAL ERROR OVERRIDE OPTION
1 ! NABOUT - ABREVIATED OUTPUT OPTION PARAMETER
Expand Down
2 changes: 1 addition & 1 deletion tests/data/reference/adcirc/stampede2_shinnecock_ike/spinup/fort.15
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
created on 2021-07-08 16:10 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
created on 2021-07-23 12:05 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
Shinacock Inlet Coarse Grid ! RUNID - 24 CHARACTER ALPANUMERIC RUN IDENTIFICATION
1 ! NFOVER - NONFATAL ERROR OVERRIDE OPTION
1 ! NABOUT - ABREVIATED OUTPUT OPTION PARAMETER
Expand Down
5 changes: 0 additions & 5 deletions tests/data/reference/nems_adcirc/hera_shinnecock_ike/configure_nems.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,6 @@
"modeled_start_time": "2008-08-23 00:00:00",
"modeled_end_time": "2008-09-06 12:00:00",
"interval": 3600.0,
"models": [
"OCN_model: adcirc\nOCN_petlist_bounds: 2 601\nOCN_attributes::\n Verbosity = off\n::",
"ATM_model: atmesh\nATM_petlist_bounds: 0 0\nATM_attributes::\n Verbosity = off\n::",
"WAV_model: ww3data\nWAV_petlist_bounds: 1 1\nWAV_attributes::\n Verbosity = off\n::"
],
"connections": [
[
"ATM -> OCN"
Expand Down
4 changes: 2 additions & 2 deletions tests/data/reference/nems_adcirc/hera_shinnecock_ike/runs/unperturbed/config.rc
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# `config.rc` generated with NEMSpy 1.0.1
atm_dir: ../../../../../../../../../input/shinnecock/ike/forcings
atm_dir: ../../../../../input/shinnecock/ike/forcings
atm_nam: wind_atm_fin_ch_time_vec.nc
wav_dir: ../../../../../../../../../input/shinnecock/ike/forcings
wav_dir: ../../../../../input/shinnecock/ike/forcings
wav_nam: ww3.Constant.20151214_sxy_ike_date.nc
2 changes: 1 addition & 1 deletion tests/data/reference/nems_adcirc/hera_shinnecock_ike/runs/unperturbed/fort.15
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
created on 2021-07-08 16:10 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
created on 2021-07-23 12:04 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
Shinacock Inlet Coarse Grid ! RUNID - 24 CHARACTER ALPANUMERIC RUN IDENTIFICATION
1 ! NFOVER - NONFATAL ERROR OVERRIDE OPTION
1 ! NABOUT - ABREVIATED OUTPUT OPTION PARAMETER
Expand Down
2 changes: 1 addition & 1 deletion tests/data/reference/nems_adcirc/hera_shinnecock_ike/spinup/fort.15
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
created on 2021-07-08 16:09 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
created on 2021-07-23 12:04 ! RUNDES - 32 CHARACTER ALPHANUMERIC RUN DESCRIPTION
Shinacock Inlet Coarse Grid ! RUNID - 24 CHARACTER ALPANUMERIC RUN IDENTIFICATION
1 ! NFOVER - NONFATAL ERROR OVERRIDE OPTION
1 ! NABOUT - ABREVIATED OUTPUT OPTION PARAMETER
Expand Down
Loading

0 comments on commit e153bdc

Please sign in to comment.