From a39c23f64cfc6cbf72e0446167c3e2f4e4224a66 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Mon, 17 Jun 2024 14:18:02 +0000 Subject: [PATCH 1/6] rename definition to entity_model --- src/ibek/entity_factory.py | 19 +-- src/ibek/{definition.py => entity_model.py} | 4 +- src/ibek/gen_scripts.py | 2 +- src/ibek/ioc.py | 110 +++++++++++------- src/ibek/params.py | 8 +- src/ibek/render.py | 2 +- src/ibek/render_db.py | 2 +- src/ibek/runtime_cmds/commands.py | 2 +- src/ibek/support.py | 4 +- .../samples/support/listarg.ibek.support.yaml | 2 + tests/test_unit.py | 6 +- 11 files changed, 87 insertions(+), 74 deletions(-) rename src/ibek/{definition.py => entity_model.py} (97%) diff --git a/src/ibek/entity_factory.py b/src/ibek/entity_factory.py index 1f154994e..1c0887b9c 100644 --- a/src/ibek/entity_factory.py +++ b/src/ibek/entity_factory.py @@ -16,7 +16,7 @@ from .ioc import Entity, EnumVal, clear_entity_model_ids from .params import Define, EnumParam, IdParam, ObjectParam -from .support import EntityDefinition, Support +from .support import EntityModel, Support from .utils import UTILS @@ -56,19 +56,12 @@ def make_entity_models(self, definition_yaml: List[Path]) -> List[Type[Entity]]: return list(self._entity_models.values()) def _make_entity_model( - self, definition: EntityDefinition, support: Support + self, definition: EntityModel, support: Support ) -> Type[Entity]: """ Create an Entity Model from a Definition instance and a Support instance. """ - def add_defines(s: dict[str, Define]) -> None: - if s: - # add in the pre_defines or post_defines as Args in the Entity - for name, value in s.items(): - typ = getattr(builtins, str(value.type)) - add_arg(name, typ, value.description, value.value) - def add_arg(name, typ, description, default): if default is None: default = PydanticUndefined @@ -100,11 +93,6 @@ def add_arg(name, typ, description, default): # fully qualified name of the Entity class including support module full_name = f"{support.module}.{definition.name}" - # add in the calculated values Jinja Templates as Fields in the Entity - # these are the pre_values that should be Jinja rendered before any - # Args (or post values) - add_defines(definition.pre_defines) - # add in each of the arguments as a Field in the Entity for name, arg in definition.params.items(): type: Any @@ -132,9 +120,6 @@ def add_arg(name, typ, description, default): type = getattr(builtins, arg.type) add_arg(name, type, arg.description, getattr(arg, "default")) # type: ignore - # add in the calculated values Jinja Templates as Fields in the Entity - add_defines(definition.post_defines) - # add the type literal which discriminates between the different Entity classes typ = Literal[full_name] # type: ignore args["type"] = (typ, Field(description=definition.description)) diff --git a/src/ibek/definition.py b/src/ibek/entity_model.py similarity index 97% rename from src/ibek/definition.py rename to src/ibek/entity_model.py index 3d4299dc3..8e1c12076 100644 --- a/src/ibek/definition.py +++ b/src/ibek/entity_model.py @@ -121,9 +121,9 @@ class EntityPVI(BaseSettings): ] -class EntityDefinition(BaseSettings): +class EntityModel(BaseSettings): """ - A single definition of a class of Entity that an IOC instance may instantiate + A Model for a class of Entity that an IOC instance may instantiate """ name: str = Field( diff --git a/src/ibek/gen_scripts.py b/src/ibek/gen_scripts.py index c69115c79..f32bb6f1f 100644 --- a/src/ibek/gen_scripts.py +++ b/src/ibek/gen_scripts.py @@ -9,7 +9,7 @@ from ibek.utils import UTILS -from .definition import Database +from .entity_model import Database from .globals import TEMPLATES from .ioc import Entity from .render import Render diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 58847f9bf..c86c5bfa3 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -5,18 +5,22 @@ from __future__ import annotations +import builtins import json +from collections import OrderedDict from enum import Enum from typing import Any, Dict, List, Sequence from pydantic import ( + ConfigDict, Field, model_validator, ) +from pydantic.fields import FieldInfo -from .definition import EntityDefinition +from .entity_model import EntityModel from .globals import BaseSettings -from .params import IdParam +from .params import Define, IdParam from .utils import UTILS # a global dict of all entity instances indexed by their ID @@ -54,14 +58,56 @@ class Entity(BaseSettings): entity_enabled: bool = Field( description="enable or disable this entity instance", default=True ) - __definition__: EntityDefinition + __definition__: EntityModel + + def _process_field( + self: Entity, name: str, typ: str, value: Any, ids: list[str] + ) -> Any: + """ + Process an Entitiy field - doing jinja rendering and type coercion as required + """ + + if isinstance(value, str): + # Jinja expansion always performed on string fields + value = UTILS.render(self, value) + # TODO this is a cheesy test - any better ideas please let me know + if "Union" in str(typ): + # Args that were non strings and have been rendered by Jinja + # must be coerced back into their original type + try: + # The following replace are to make the string json compatible + # (maybe we should python decode instead of json.loads) + value = value.replace("'", '"') + value = value.replace("True", "true") + value = value.replace("False", "false") + value = json.loads(value) + except: + print(f"ERROR: fail to decode {value} as a {typ}") + raise + + if typ == object: + # look up the actual object by it's id + if isinstance(value, str): + value = get_entity_by_id(value) + + # pre/post_defines are added into the model instance fields list here + if name not in self.model_fields: + self.model_fields[name] = FieldInfo(annotation=type(typ), default=value) + + # update the attribute with the rendered value + setattr(self, name, value) + + if name in ids: + # add this entity to the global id index + if value in id_to_entity: + raise ValueError(f"Duplicate id {value} in {list(id_to_entity)}") + id_to_entity[value] = self @model_validator(mode="after") def add_ibek_attributes(self): """ Whole Entity model validation """ - # find the id field in this Entity if it has one ids = { name @@ -69,45 +115,25 @@ def add_ibek_attributes(self): if isinstance(value, IdParam) } - entity_dict = self.model_dump() - for arg, value in entity_dict.items(): - model_field = self.model_fields[arg] + # Do jinja rendering of pre_defines/ parameters / post_defines + # in the correct order. _process_parameter also adds the pre/post_defines + # into the model instance itself so that they are available later in the + # second phase of jinja rendering for pre_init, post_init and databases. - if isinstance(value, str): - # Jinja expansion of any of the Entity's string args/values - value = UTILS.render(entity_dict, value) - # this is a cheesy test - any better ideas please let me know - if "Union" in str(model_field.annotation): - # Args that were non strings and have been rendered by Jinja - # must be coerced back into their original type - try: - # The following replace are to make the string json compatible - # (maybe we should python decode instead of json.loads) - value = value.replace("'", '"') - value = value.replace("True", "true") - value = value.replace("False", "false") - value = json.loads(value) - except: - print( - f"ERROR: fail to decode {value} as a {model_field.annotation}" - ) - raise - - if model_field.annotation == object: - # look up the actual object by it's id - if isinstance(value, str): - value = get_entity_by_id(value) - - # update this entity instance with the rendered value - setattr(self, arg, value) - # update the entity_dict with the rendered value - entity_dict[arg] = value - - if arg in ids: - # add this entity to the global id index - if value in id_to_entity: - raise ValueError(f"Duplicate id {value} in {list(id_to_entity)}") - id_to_entity[value] = self + for name, define in self.__definition__.pre_defines.items(): + self._process_field(name, define.value, define.type, ids) + + for name, parameter in self.model_dump().items(): + typ = self.model_fields[name].annotation + self._process_field(name, parameter, typ, ids) + + for name, define in self.__definition__.post_defines.items(): + self._process_field(name, define.value, define.type, ids) + + # we have updated the model with jinja rendered values and also with + # pre/post_defines so allow extras and rebuild the model + self.model_config["extra"] = "allow" + self.model_rebuild(force=True) return self diff --git a/src/ibek/params.py b/src/ibek/params.py index f069a230b..8831caebe 100644 --- a/src/ibek/params.py +++ b/src/ibek/params.py @@ -17,7 +17,7 @@ ] -class DefinesTypes(Enum): +class DefineTypes(Enum): """The type of a value""" string = "str" @@ -34,14 +34,14 @@ def __repr__(self): class Define(BaseSettings): - """A calculated string value for a definition""" + """A calculated value for an Entity Model""" description: str = Field( description="Description of what the value will be used for" ) value: Any = Field(description="The contents of the value") - type: DefinesTypes = Field( - description="The type of the value", default=DefinesTypes.string + type: DefineTypes | None = Field( + description="The type of the value", default=DefineTypes.string ) diff --git a/src/ibek/render.py b/src/ibek/render.py index 8782fc311..649d541bd 100644 --- a/src/ibek/render.py +++ b/src/ibek/render.py @@ -4,7 +4,7 @@ from typing import Callable, List, Optional, Sequence, Union -from .definition import Comment, Script, Text, When +from .entity_model import Comment, Script, Text, When from .ioc import Entity from .utils import UTILS diff --git a/src/ibek/render_db.py b/src/ibek/render_db.py index 120f77e23..7e1bced90 100644 --- a/src/ibek/render_db.py +++ b/src/ibek/render_db.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple -from ibek.definition import Database +from ibek.entity_model import Database from ibek.ioc import Entity from ibek.utils import UTILS diff --git a/src/ibek/runtime_cmds/commands.py b/src/ibek/runtime_cmds/commands.py index 89f5fe553..f77af8b96 100644 --- a/src/ibek/runtime_cmds/commands.py +++ b/src/ibek/runtime_cmds/commands.py @@ -8,8 +8,8 @@ from pvi._format.template import format_template from pvi.device import Device -from ibek.definition import Database from ibek.entity_factory import EntityFactory +from ibek.entity_model import Database from ibek.gen_scripts import create_boot_script, create_db_script from ibek.globals import GLOBALS, NaturalOrderGroup from ibek.ioc import IOC, Entity diff --git a/src/ibek/support.py b/src/ibek/support.py index fee58d870..33239ba97 100644 --- a/src/ibek/support.py +++ b/src/ibek/support.py @@ -9,7 +9,7 @@ from pydantic import Field -from .definition import EntityDefinition +from .entity_model import EntityModel from .globals import BaseSettings @@ -26,7 +26,7 @@ class Support(BaseSettings): ) module: str = Field(description="Support module name, normally the repo name") - defs: Sequence[EntityDefinition] = Field( + defs: Sequence[EntityModel] = Field( description="The definitions an IOC can create using this module" ) diff --git a/tests/samples/support/listarg.ibek.support.yaml b/tests/samples/support/listarg.ibek.support.yaml index a0611cc0b..378fe20fa 100644 --- a/tests/samples/support/listarg.ibek.support.yaml +++ b/tests/samples/support/listarg.ibek.support.yaml @@ -36,6 +36,8 @@ defs: name: description: |- name of the character + type: str # TODO - do we want to enforce this ? at present schema allows + # no value for type and should default to str but the code fails. value: |- Dave Lister diff --git a/tests/test_unit.py b/tests/test_unit.py index 9df46f131..d8855b816 100644 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -11,7 +11,7 @@ from ibek.ioc import id_to_entity from ibek.ioc_factory import IocFactory from ibek.params import IdParam, ObjectParam -from ibek.support import EntityDefinition, Support +from ibek.support import EntityModel, Support from ibek.utils import UTILS @@ -22,12 +22,12 @@ def test_object_references(entity_factory): support = Support( module="mymodule", defs=[ - EntityDefinition( + EntityModel( name="port", description="a port", params={"name": IdParam(description="an id")}, ), - EntityDefinition( + EntityModel( name="device", description="a device", params={"port": ObjectParam(description="the port")}, From 962ffd6467bd705826c56b10b1e226ea9abe8797 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Mon, 17 Jun 2024 20:23:42 +0000 Subject: [PATCH 2/6] test_list working --- src/ibek/ioc.py | 45 +++++++++---------- .../samples/support/listarg.ibek.support.yaml | 14 +++--- 2 files changed, 28 insertions(+), 31 deletions(-) diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index c86c5bfa3..242988fee 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -5,14 +5,12 @@ from __future__ import annotations +import ast import builtins -import json -from collections import OrderedDict from enum import Enum from typing import Any, Dict, List, Sequence from pydantic import ( - ConfigDict, Field, model_validator, ) @@ -20,7 +18,7 @@ from .entity_model import EntityModel from .globals import BaseSettings -from .params import Define, IdParam +from .params import IdParam from .utils import UTILS # a global dict of all entity instances indexed by their ID @@ -61,28 +59,27 @@ class Entity(BaseSettings): __definition__: EntityModel def _process_field( - self: Entity, name: str, typ: str, value: Any, ids: list[str] + self: Entity, name: str, value: Any, typ_str: str, ids: list[str] ) -> Any: """ - Process an Entitiy field - doing jinja rendering and type coercion as required + Process an Entity field - doing jinja rendering and type coercion as required """ + # these fields don't need any rendering + if name in ["type", "entity_enabled"]: + return value + + typ = getattr(builtins, typ_str) + if isinstance(value, str): # Jinja expansion always performed on string fields value = UTILS.render(self, value) - # TODO this is a cheesy test - any better ideas please let me know - if "Union" in str(typ): - # Args that were non strings and have been rendered by Jinja - # must be coerced back into their original type + if typ is not str: + # coerce the rendered parameter to its intended type try: - # The following replace are to make the string json compatible - # (maybe we should python decode instead of json.loads) - value = value.replace("'", '"') - value = value.replace("True", "true") - value = value.replace("False", "false") - value = json.loads(value) + value = typ(ast.literal_eval(value)) except: - print(f"ERROR: fail to decode {value} as a {typ}") + print(f"ERROR: decoding field '{name}', value '{value}' as {typ}") raise if typ == object: @@ -92,7 +89,7 @@ def _process_field( # pre/post_defines are added into the model instance fields list here if name not in self.model_fields: - self.model_fields[name] = FieldInfo(annotation=type(typ), default=value) + self.model_fields[name] = FieldInfo(annotation=typ, default=value) # update the attribute with the rendered value setattr(self, name, value) @@ -116,16 +113,16 @@ def add_ibek_attributes(self): } # Do jinja rendering of pre_defines/ parameters / post_defines - # in the correct order. _process_parameter also adds the pre/post_defines - # into the model instance itself so that they are available later in the - # second phase of jinja rendering for pre_init, post_init and databases. + # in the correct order. self._process_field also adds the field to this model + # instance if it does not already exist. Hence pre/post_defines are added to + # the model instance and are available for the phase 2 (final) jinja + # rendering performed in ibek.commands.generate(). for name, define in self.__definition__.pre_defines.items(): self._process_field(name, define.value, define.type, ids) - for name, parameter in self.model_dump().items(): - typ = self.model_fields[name].annotation - self._process_field(name, parameter, typ, ids) + for name, parameter in self.__definition__.params.items(): + self._process_field(name, getattr(self, name), parameter.type, ids) for name, define in self.__definition__.post_defines.items(): self._process_field(name, define.value, define.type, ids) diff --git a/tests/samples/support/listarg.ibek.support.yaml b/tests/samples/support/listarg.ibek.support.yaml index 378fe20fa..180e2d6b9 100644 --- a/tests/samples/support/listarg.ibek.support.yaml +++ b/tests/samples/support/listarg.ibek.support.yaml @@ -41,6 +41,13 @@ defs: value: |- Dave Lister + cryo_years: + description: |- + years in cryogenic sleep + type: int + value: 3000000 + # https://en.wikipedia.org/wiki/Dave_Lister#:~:text=As%20a%20result%2C%20Holly%20keeps,radiation%20levels%20return%20to%20normal. + params: # example of using aliases to merge multiple dictionaries into this one <<: [*params1, *params2] @@ -54,13 +61,6 @@ defs: {{ name == 'Dave Lister' }} post_defines: - cryo_years: - description: |- - years in cryogenic sleep - type: int - value: 3000000 - # https://en.wikipedia.org/wiki/Dave_Lister#:~:text=As%20a%20result%2C%20Holly%20keeps,radiation%20levels%20return%20to%20normal. - # example type list value not using Jinja friends: description: |- From 99d6b30828f7c38025ecd72218c2406858366e1a Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Mon, 17 Jun 2024 20:58:22 +0000 Subject: [PATCH 3/6] all tests passing --- src/ibek/ioc.py | 42 +++++------ .../schemas/fastVacuum.ibek.ioc.schema.json | 74 +------------------ .../schemas/gauges.ibek.ioc.schema.json | 6 +- .../samples/schemas/ibek.support.schema.json | 21 +++--- .../samples/schemas/ipac.ibek.ioc.schema.json | 12 --- .../samples/schemas/listarg.ibek.schema.json | 61 --------------- .../schemas/technosoft.ibek.ioc.schema.json | 29 -------- .../schemas/utils.ibek.ioc.schema.json | 12 --- .../support/fastVacuum.ibek.support.yaml | 2 + 9 files changed, 36 insertions(+), 223 deletions(-) diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 242988fee..92c5f950f 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -58,31 +58,29 @@ class Entity(BaseSettings): ) __definition__: EntityModel - def _process_field( - self: Entity, name: str, value: Any, typ_str: str, ids: list[str] - ) -> Any: + def _process_field(self: Entity, name: str, value: Any, typ: str): """ - Process an Entity field - doing jinja rendering and type coercion as required + Process an Entity field - doing jinja rendering, type coercion and + object id storing/lookup as required. """ # these fields don't need any rendering if name in ["type", "entity_enabled"]: - return value - - typ = getattr(builtins, typ_str) + return if isinstance(value, str): # Jinja expansion always performed on string fields value = UTILS.render(self, value) - if typ is not str: + if typ in ["list", "int", "float", "bool"]: # coerce the rendered parameter to its intended type try: - value = typ(ast.literal_eval(value)) + cast_type = getattr(builtins, typ) + value = cast_type(ast.literal_eval(value)) except: print(f"ERROR: decoding field '{name}', value '{value}' as {typ}") raise - if typ == object: + if typ == "object": # look up the actual object by it's id if isinstance(value, str): value = get_entity_by_id(value) @@ -94,7 +92,7 @@ def _process_field( # update the attribute with the rendered value setattr(self, name, value) - if name in ids: + if typ == "id": # add this entity to the global id index if value in id_to_entity: raise ValueError(f"Duplicate id {value} in {list(id_to_entity)}") @@ -105,12 +103,6 @@ def add_ibek_attributes(self): """ Whole Entity model validation """ - # find the id field in this Entity if it has one - ids = { - name - for name, value in self.__definition__.params.items() - if isinstance(value, IdParam) - } # Do jinja rendering of pre_defines/ parameters / post_defines # in the correct order. self._process_field also adds the field to this model @@ -118,19 +110,21 @@ def add_ibek_attributes(self): # the model instance and are available for the phase 2 (final) jinja # rendering performed in ibek.commands.generate(). - for name, define in self.__definition__.pre_defines.items(): - self._process_field(name, define.value, define.type, ids) + if self.__definition__.pre_defines: + for name, define in self.__definition__.pre_defines.items(): + self._process_field(name, define.value, define.type) - for name, parameter in self.__definition__.params.items(): - self._process_field(name, getattr(self, name), parameter.type, ids) + if self.__definition__.params: + for name, parameter in self.__definition__.params.items(): + self._process_field(name, getattr(self, name), parameter.type) - for name, define in self.__definition__.post_defines.items(): - self._process_field(name, define.value, define.type, ids) + if self.__definition__.post_defines: + for name, define in self.__definition__.post_defines.items(): + self._process_field(name, define.value, define.type) # we have updated the model with jinja rendered values and also with # pre/post_defines so allow extras and rebuild the model self.model_config["extra"] = "allow" - self.model_rebuild(force=True) return self diff --git a/tests/samples/schemas/fastVacuum.ibek.ioc.schema.json b/tests/samples/schemas/fastVacuum.ibek.ioc.schema.json index b58459a77..b23309d85 100644 --- a/tests/samples/schemas/fastVacuum.ibek.ioc.schema.json +++ b/tests/samples/schemas/fastVacuum.ibek.ioc.schema.json @@ -95,46 +95,6 @@ "default": 0, "description": "union of and jinja representation of {typ}", "title": "Timeout" - }, - "gaugeNum": { - "anyOf": [ - { - "description": "jinja that renders to ", - "pattern": ".*\\{\\{.*\\}\\}.*", - "type": "string" - }, - { - "description": "auto gauge count", - "type": "integer" - } - ], - "default": "{{ _global.incrementor(master, start=1) }}", - "description": "union of and jinja representation of {typ}", - "title": "Gaugenum" - }, - "fan": { - "default": "{{ \"%02d\" % (gaugeNum / 7 + 1) }}", - "description": "fan number", - "title": "Fan", - "type": "string" - }, - "mask": { - "default": "{{ _global.incrementor(\"mask_{}\".format(master), 2, 2**gaugeNum) }}", - "description": "mask for the channel", - "title": "Mask", - "type": "string" - }, - "lnk_no": { - "default": "{{ ((gaugeNum - 1) % 6) + 1 }}", - "description": "link number", - "title": "Lnk No", - "type": "string" - }, - "gaugePV": { - "default": "{{ master.device }}:GAUGE:{{id}}_0", - "description": "Gauge PV", - "title": "Gaugepv", - "type": "string" } }, "required": [ @@ -186,41 +146,9 @@ }, "device": { "default": "{{dom}}-VA-FAST-01", - "description": "device name", + "description": "Device prefix", "title": "Device", "type": "string" - }, - "waveform_nelm": { - "anyOf": [ - { - "description": "jinja that renders to ", - "pattern": ".*\\{\\{.*\\}\\}.*", - "type": "string" - }, - { - "description": "waveform element count", - "type": "integer" - } - ], - "default": 500, - "description": "union of and jinja representation of {typ}", - "title": "Waveform Nelm" - }, - "combined_nelm": { - "anyOf": [ - { - "description": "jinja that renders to ", - "pattern": ".*\\{\\{.*\\}\\}.*", - "type": "string" - }, - { - "description": "total waveform element count", - "type": "integer" - } - ], - "default": "{{ 6 * waveform_nelm }}", - "description": "union of and jinja representation of {typ}", - "title": "Combined Nelm" } }, "required": [ diff --git a/tests/samples/schemas/gauges.ibek.ioc.schema.json b/tests/samples/schemas/gauges.ibek.ioc.schema.json index 6230cda5a..5a18c93b9 100644 --- a/tests/samples/schemas/gauges.ibek.ioc.schema.json +++ b/tests/samples/schemas/gauges.ibek.ioc.schema.json @@ -212,17 +212,17 @@ "title": "Gauge1" }, "gauge2": { - "default": "{{gauge1.name}}", + "default": "{{gauge1}}", "description": "Second gauge", "title": "Gauge2" }, "gauge3": { - "default": "{{gauge1.name}}\n", + "default": "{{gauge1}}\n", "description": "Third gauge", "title": "Gauge3" }, "gauge4": { - "default": "{{gauge1.name}}\n", + "default": "{{gauge1}}\n", "description": "Fourth gauge", "title": "Gauge4" } diff --git a/tests/samples/schemas/ibek.support.schema.json b/tests/samples/schemas/ibek.support.schema.json index b1dc86889..1af29ed48 100644 --- a/tests/samples/schemas/ibek.support.schema.json +++ b/tests/samples/schemas/ibek.support.schema.json @@ -114,7 +114,7 @@ }, "Define": { "additionalProperties": false, - "description": "A calculated string value for a definition", + "description": "A calculated value for an Entity Model", "properties": { "description": { "description": "Description of what the value will be used for", @@ -126,9 +126,12 @@ "title": "Value" }, "type": { - "allOf": [ + "anyOf": [ { - "$ref": "#/$defs/DefinesTypes" + "$ref": "#/$defs/DefineTypes" + }, + { + "type": "null" } ], "default": "str", @@ -142,7 +145,7 @@ "title": "Define", "type": "object" }, - "DefinesTypes": { + "DefineTypes": { "description": "The type of a value", "enum": [ "str", @@ -151,12 +154,12 @@ "bool", "list" ], - "title": "DefinesTypes", + "title": "DefineTypes", "type": "string" }, - "EntityDefinition": { + "EntityModel": { "additionalProperties": false, - "description": "A single definition of a class of Entity that an IOC instance may instantiate", + "description": "A Model for a class of Entity that an IOC instance may instantiate", "properties": { "name": { "description": "Publish Definition as type . for IOC instances", @@ -313,7 +316,7 @@ "name", "description" ], - "title": "EntityDefinition", + "title": "EntityModel", "type": "object" }, "EntityPVI": { @@ -704,7 +707,7 @@ "defs": { "description": "The definitions an IOC can create using this module", "items": { - "$ref": "#/$defs/EntityDefinition" + "$ref": "#/$defs/EntityModel" }, "title": "Defs", "type": "array" diff --git a/tests/samples/schemas/ipac.ibek.ioc.schema.json b/tests/samples/schemas/ipac.ibek.ioc.schema.json index 4ea7b3b1e..173bf6a5b 100644 --- a/tests/samples/schemas/ipac.ibek.ioc.schema.json +++ b/tests/samples/schemas/ipac.ibek.ioc.schema.json @@ -176,12 +176,6 @@ "default": 1, "description": "union of and jinja representation of {typ}", "title": "Count" - }, - "number": { - "default": "{{ _global.incrementor(\"InterruptVector\", start=192, stop=255, increment=count) }}", - "description": "Interrupt Vector number", - "title": "Number", - "type": "string" } }, "required": [ @@ -535,12 +529,6 @@ "interrupt_vector": { "description": "Interrupt Vector reserved with epics.InterruptVectorVME, count=1", "title": "Interrupt Vector" - }, - "card_id": { - "default": "{{ _global.incrementor(\"Carriers\", start=0) }}", - "description": "Carrier Card Identifier", - "title": "Card Id", - "type": "string" } }, "required": [ diff --git a/tests/samples/schemas/listarg.ibek.schema.json b/tests/samples/schemas/listarg.ibek.schema.json index ff09d96f3..ecb817a57 100644 --- a/tests/samples/schemas/listarg.ibek.schema.json +++ b/tests/samples/schemas/listarg.ibek.schema.json @@ -18,12 +18,6 @@ "title": "Entity Enabled", "type": "boolean" }, - "name": { - "default": "Dave Lister", - "description": "name of the character", - "title": "Name", - "type": "string" - }, "age": { "anyOf": [ { @@ -77,61 +71,6 @@ "default": "{{ name == 'Dave Lister' }}", "description": "union of and jinja representation of {typ}", "title": "Is Human" - }, - "cryo_years": { - "anyOf": [ - { - "description": "jinja that renders to ", - "pattern": ".*\\{\\{.*\\}\\}.*", - "type": "string" - }, - { - "description": "years in cryogenic sleep", - "type": "integer" - } - ], - "default": 3000000, - "description": "union of and jinja representation of {typ}", - "title": "Cryo Years" - }, - "friends": { - "anyOf": [ - { - "description": "jinja that renders to ", - "pattern": ".*\\{\\{.*\\}\\}.*", - "type": "string" - }, - { - "description": "friends of Lister", - "items": {}, - "type": "array" - } - ], - "default": [ - "Rimmer", - "Cat", - "Kryten", - "Holly" - ], - "description": "union of and jinja representation of {typ}", - "title": "Friends" - }, - "time_vortex": { - "anyOf": [ - { - "description": "jinja that renders to ", - "pattern": ".*\\{\\{.*\\}\\}.*", - "type": "string" - }, - { - "description": "friends after the Time Hole episode", - "items": {}, - "type": "array" - } - ], - "default": "{{ (friends * 3) | sort }}", - "description": "union of and jinja representation of {typ}", - "title": "Time Vortex" } }, "required": [ diff --git a/tests/samples/schemas/technosoft.ibek.ioc.schema.json b/tests/samples/schemas/technosoft.ibek.ioc.schema.json index 5b8960761..8642cd8be 100644 --- a/tests/samples/schemas/technosoft.ibek.ioc.schema.json +++ b/tests/samples/schemas/technosoft.ibek.ioc.schema.json @@ -61,29 +61,6 @@ "description": "TML Configuration", "title": "Config", "type": "string" - }, - "axisConfiguration": { - "anyOf": [ - { - "description": "jinja that renders to ", - "pattern": ".*\\{\\{.*\\}\\}.*", - "type": "string" - }, - { - "description": "collects the axis configuration from axis entities", - "items": {}, - "type": "array" - } - ], - "default": [], - "description": "union of and jinja representation of {typ}", - "title": "Axisconfiguration" - }, - "axisNum": { - "default": "{{ \"axes_{}\".format(controllerName) }}", - "description": "A name for the axis counter (used to generate unique axis numbers)", - "title": "Axisnum", - "type": "string" } }, "required": [ @@ -433,12 +410,6 @@ "description": "Axis configuration string to add to the controller configuration", "title": "Config", "type": "string" - }, - "axisConfiguration": { - "default": "{{ controller.axisConfiguration.append(CONFIG) }}", - "description": "Adds an axis configuration entry to the controller's list", - "title": "Axisconfiguration", - "type": "string" } }, "required": [ diff --git a/tests/samples/schemas/utils.ibek.ioc.schema.json b/tests/samples/schemas/utils.ibek.ioc.schema.json index 033fc5ac9..7662b55e7 100644 --- a/tests/samples/schemas/utils.ibek.ioc.schema.json +++ b/tests/samples/schemas/utils.ibek.ioc.schema.json @@ -22,18 +22,6 @@ "description": "A name for an interrupt vector variable", "title": "Name", "type": "string" - }, - "test_global_var": { - "default": "{{ _global.set(\"magic_global\", 42) }}", - "description": "test global variable setter", - "title": "Test Global Var", - "type": "string" - }, - "get_global": { - "default": "{{ _global.get(\"magic_global\") }}", - "description": "test global variable getter", - "title": "Get Global", - "type": "string" } }, "required": [ diff --git a/tests/samples/support/fastVacuum.ibek.support.yaml b/tests/samples/support/fastVacuum.ibek.support.yaml index bcd96c779..c663dcc0a 100644 --- a/tests/samples/support/fastVacuum.ibek.support.yaml +++ b/tests/samples/support/fastVacuum.ibek.support.yaml @@ -40,6 +40,8 @@ defs: type: str description: |- Device prefix + default: |- + {{dom}}-VA-FAST-01 post_defines: device: From ffe9df164e2146ed5cb7ff07fdfc3b833d67628f Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Mon, 17 Jun 2024 21:02:58 +0000 Subject: [PATCH 4/6] lint --- src/ibek/entity_factory.py | 2 +- src/ibek/ioc.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ibek/entity_factory.py b/src/ibek/entity_factory.py index 1c0887b9c..2bdf14c12 100644 --- a/src/ibek/entity_factory.py +++ b/src/ibek/entity_factory.py @@ -15,7 +15,7 @@ from ibek.globals import JINJA from .ioc import Entity, EnumVal, clear_entity_model_ids -from .params import Define, EnumParam, IdParam, ObjectParam +from .params import EnumParam, IdParam, ObjectParam from .support import EntityModel, Support from .utils import UTILS diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 92c5f950f..c08aca2d6 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -18,7 +18,6 @@ from .entity_model import EntityModel from .globals import BaseSettings -from .params import IdParam from .utils import UTILS # a global dict of all entity instances indexed by their ID @@ -87,7 +86,7 @@ def _process_field(self: Entity, name: str, value: Any, typ: str): # pre/post_defines are added into the model instance fields list here if name not in self.model_fields: - self.model_fields[name] = FieldInfo(annotation=typ, default=value) + self.model_fields[name] = FieldInfo(annotation=str, default=value) # update the attribute with the rendered value setattr(self, name, value) From 25ed7cf47c69df4956dff331f83a07b786cbf917 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 18 Jun 2024 05:27:02 +0000 Subject: [PATCH 5/6] remove redundant changes --- src/ibek/ioc.py | 65 ++++++++----------- .../samples/support/listarg.ibek.support.yaml | 2 - 2 files changed, 28 insertions(+), 39 deletions(-) diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index c08aca2d6..87acd9bbe 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -20,22 +20,6 @@ from .globals import BaseSettings from .utils import UTILS -# a global dict of all entity instances indexed by their ID -id_to_entity: Dict[str, Entity] = {} - - -def get_entity_by_id(id: str) -> Entity: - try: - return id_to_entity[id] - except KeyError: - raise ValueError(f"object {id} not found in {list(id_to_entity)}") - - -def clear_entity_model_ids(): - """Resets the global id_to_entity dict""" - - id_to_entity.clear() - class EnumVal(Enum): """ @@ -57,16 +41,22 @@ class Entity(BaseSettings): ) __definition__: EntityModel + def __init__(self, **data): + super().__init__(**data) + self.__id_to_entity = {} + + def get_entity_by_id(self, id: str) -> Entity: + try: + return self.__id_to_entity[id] + except KeyError: + raise ValueError(f"object {id} not found in {list(self.__id_to_entity)}") + def _process_field(self: Entity, name: str, value: Any, typ: str): """ Process an Entity field - doing jinja rendering, type coercion and - object id storing/lookup as required. + object id storage/lookup as required. """ - # these fields don't need any rendering - if name in ["type", "entity_enabled"]: - return - if isinstance(value, str): # Jinja expansion always performed on string fields value = UTILS.render(self, value) @@ -82,32 +72,34 @@ def _process_field(self: Entity, name: str, value: Any, typ: str): if typ == "object": # look up the actual object by it's id if isinstance(value, str): - value = get_entity_by_id(value) + value = self.get_entity_by_id(value) - # pre/post_defines are added into the model instance fields list here + # If this field is not pre-existing, add it into the model instance. + # This is how pre/post_defines are added. if name not in self.model_fields: self.model_fields[name] = FieldInfo(annotation=str, default=value) - # update the attribute with the rendered value + # update the model instance attribute with the rendered value setattr(self, name, value) if typ == "id": # add this entity to the global id index - if value in id_to_entity: - raise ValueError(f"Duplicate id {value} in {list(id_to_entity)}") - id_to_entity[value] = self + if value in self.__id_to_entity: + raise ValueError(f"Duplicate id {value} in {list(self.__id_to_entity)}") + self.__id_to_entity[value] = self @model_validator(mode="after") def add_ibek_attributes(self): """ Whole Entity model validation - """ - # Do jinja rendering of pre_defines/ parameters / post_defines - # in the correct order. self._process_field also adds the field to this model - # instance if it does not already exist. Hence pre/post_defines are added to - # the model instance and are available for the phase 2 (final) jinja - # rendering performed in ibek.commands.generate(). + Do jinja rendering of pre_defines/ parameters / post_defines + in the correct order. + + Also adds pre_define and post_defines to the model instance, making + them available for the phase 2 (final) jinja rendering performed in + ibek.runtime_cmds.generate(). + """ if self.__definition__.pre_defines: for name, define in self.__definition__.pre_defines.items(): @@ -121,10 +113,6 @@ def add_ibek_attributes(self): for name, define in self.__definition__.post_defines.items(): self._process_field(name, define.value, define.type) - # we have updated the model with jinja rendered values and also with - # pre/post_defines so allow extras and rebuild the model - self.model_config["extra"] = "allow" - return self def __str__(self): @@ -148,3 +136,6 @@ class IOC(BaseSettings): description="A place to create any anchors required for repeating YAML", default=(), ) + + # a dict of all entity instances in this IOC, indexed by their ID + __id_to_entity: Dict[str, Entity] diff --git a/tests/samples/support/listarg.ibek.support.yaml b/tests/samples/support/listarg.ibek.support.yaml index 180e2d6b9..ded280126 100644 --- a/tests/samples/support/listarg.ibek.support.yaml +++ b/tests/samples/support/listarg.ibek.support.yaml @@ -36,8 +36,6 @@ defs: name: description: |- name of the character - type: str # TODO - do we want to enforce this ? at present schema allows - # no value for type and should default to str but the code fails. value: |- Dave Lister From 263df5bd4f6858548364cd6f64b3c103c0f08736 Mon Sep 17 00:00:00 2001 From: Giles Knap Date: Tue, 18 Jun 2024 05:46:49 +0000 Subject: [PATCH 6/6] revert id_to_entity changes --- src/ibek/ioc.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/ibek/ioc.py b/src/ibek/ioc.py index 87acd9bbe..fc445682e 100644 --- a/src/ibek/ioc.py +++ b/src/ibek/ioc.py @@ -20,6 +20,22 @@ from .globals import BaseSettings from .utils import UTILS +# a global dict of all entity instances indexed by their ID +id_to_entity: Dict[str, Entity] = {} + + +def get_entity_by_id(id: str) -> Entity: + try: + return id_to_entity[id] + except KeyError: + raise ValueError(f"object {id} not found in {list(id_to_entity)}") + + +def clear_entity_model_ids(): + """Resets the global id_to_entity dict""" + + id_to_entity.clear() + class EnumVal(Enum): """ @@ -41,16 +57,6 @@ class Entity(BaseSettings): ) __definition__: EntityModel - def __init__(self, **data): - super().__init__(**data) - self.__id_to_entity = {} - - def get_entity_by_id(self, id: str) -> Entity: - try: - return self.__id_to_entity[id] - except KeyError: - raise ValueError(f"object {id} not found in {list(self.__id_to_entity)}") - def _process_field(self: Entity, name: str, value: Any, typ: str): """ Process an Entity field - doing jinja rendering, type coercion and @@ -72,7 +78,7 @@ def _process_field(self: Entity, name: str, value: Any, typ: str): if typ == "object": # look up the actual object by it's id if isinstance(value, str): - value = self.get_entity_by_id(value) + value = get_entity_by_id(value) # If this field is not pre-existing, add it into the model instance. # This is how pre/post_defines are added. @@ -84,9 +90,9 @@ def _process_field(self: Entity, name: str, value: Any, typ: str): if typ == "id": # add this entity to the global id index - if value in self.__id_to_entity: - raise ValueError(f"Duplicate id {value} in {list(self.__id_to_entity)}") - self.__id_to_entity[value] = self + if value in id_to_entity: + raise ValueError(f"Duplicate id {value} in {list(id_to_entity)}") + id_to_entity[value] = self @model_validator(mode="after") def add_ibek_attributes(self): @@ -136,6 +142,3 @@ class IOC(BaseSettings): description="A place to create any anchors required for repeating YAML", default=(), ) - - # a dict of all entity instances in this IOC, indexed by their ID - __id_to_entity: Dict[str, Entity]