Start working on the new Xmile translator to AM

pysd/building/python/visitors.py → pysd/building/python/python_expressions_builder.py

@@ -1,11 +1,10 @@
-from re import X
 import warnings
 from dataclasses import dataclass
 
 import numpy as np
 from pysd.py_backend.utils import compute_shape
 
-from pysd.translation.structures import components as ct
+from pysd.translation.structures import abstract_expressions as ae
 from .python_functions import functionspace
 
 
@@ -435,6 +434,7 @@ def build(self, arguments):
                 subscripts=final_subs,
                 order=0)
 
+
 class ExtDataBuilder(StructureBuilder):
     def __init__(self, getdata_str,  component):
         super().__init__(None, component)
@@ -728,7 +728,8 @@ def build(self, arguments):
         self.component.subtype = "Smooth"
         self.section.imports.add("statefuls", "Smooth")
         arguments["input"].reshape(self.section.subscripts, self.def_subs)
-        arguments["smooth_time"].reshape(self.section.subscripts, self.def_subs)
+        arguments["smooth_time"].reshape(
+            self.section.subscripts, self.def_subs)
         arguments["initial"].reshape(self.section.subscripts, self.def_subs)
         arguments["name"] = self.section.namespace.make_python_identifier(
             self.element.identifier, prefix="_smooth")
@@ -775,7 +776,8 @@ def build(self, arguments):
         self.component.subtype = "Trend"
         self.section.imports.add("statefuls", "Trend")
         arguments["input"].reshape(self.section.subscripts, self.def_subs)
-        arguments["average_time"].reshape(self.section.subscripts, self.def_subs)
+        arguments["average_time"].reshape(
+            self.section.subscripts, self.def_subs)
         arguments["initial"].reshape(self.section.subscripts, self.def_subs)
         arguments["name"] = self.section.namespace.make_python_identifier(
             self.element.identifier, prefix="_trend")
@@ -817,7 +819,8 @@ def build(self, arguments):
         self.component.subtype = "Forecast"
         self.section.imports.add("statefuls", "Forecast")
         arguments["input"].reshape(self.section.subscripts, self.def_subs)
-        arguments["average_time"].reshape(self.section.subscripts, self.def_subs)
+        arguments["average_time"].reshape(
+            self.section.subscripts, self.def_subs)
         arguments["horizon"].reshape(self.section.subscripts, self.def_subs)
         arguments["name"] = self.section.namespace.make_python_identifier(
             self.element.identifier, prefix="_forecast")
@@ -1041,7 +1044,8 @@ def build(self, arguments):
         original_subs = self.section.subscripts.make_coord_dict(
                     self.section.subscripts.elements[reference])
 
-        expression, final_subs = self.visit_subscripts(expression, original_subs)
+        expression, final_subs = self.visit_subscripts(
+            expression, original_subs)
 
         return BuildAST(
             expression=expression,
@@ -1061,7 +1065,8 @@ def visit_subscripts(self, expression, original_subs):
             elif len(coord) < len(orig_coord):
                 # subset a subrange
                 # NUMPY: subset value [:, :, np.array([1, 0]), :]
-                # NUMPY: as order may change we need to check if dim != orig_dim
+                # NUMPY: as order may change we need to check if
+                #        dim != orig_dim
                 # NUMPY: use also ranges [:, :, 2:5, :] when possible
                 if dim.endswith("!"):
                     loc.append("_subscript_dict['%s']" % dim[:-1])
@@ -1077,7 +1082,8 @@ def visit_subscripts(self, expression, original_subs):
                 float = False
 
             if dim != orig_dim and len(coord) != 1:
-                # NUMPY: check order of dimensions, make all subranges work with the same dimensions?
+                # NUMPY: check order of dimensions, make all subranges work
+                #        with the same dimensions?
                 # NUMPY: this could be solved in the previous if/then/else
                 rename[orig_dim] = dim
 
@@ -1187,27 +1193,27 @@ def _merge_dependencies(current, new):
 
 class ASTVisitor:
     builders = {
-        ct.InitialStructure: InitialBuilder,
-        ct.IntegStructure: IntegBuilder,
-        ct.DelayStructure: lambda x, y: DelayBuilder("Delay", x, y),
-        ct.DelayNStructure: lambda x, y: DelayBuilder("DelayN", x, y),
-        ct.DelayFixedStructure: DelayFixedBuilder,
-        ct.SmoothStructure: SmoothBuilder,
-        ct.SmoothNStructure: SmoothBuilder,
-        ct.TrendStructure: TrendBuilder,
-        ct.ForecastStructure: ForecastBuilder,
-        ct.SampleIfTrueStructure: SampleIfTrueBuilder,
-        ct.GetConstantsStructure: ExtConstantBuilder,
-        ct.GetDataStructure: ExtDataBuilder,
-        ct.GetLookupsStructure: ExtLookupBuilder,
-        ct.LookupsStructure: LookupsBuilder,
-        ct.InlineLookupsStructure: InlineLookupsBuilder,
-        ct.DataStructure: TabDataBuilder,
-        ct.ReferenceStructure: ReferenceBuilder,
-        ct.CallStructure: CallBuilder,
-        ct.GameStructure: GameBuilder,
-        ct.LogicStructure: OperationBuilder,
-        ct.ArithmeticStructure: OperationBuilder,
+        ae.InitialStructure: InitialBuilder,
+        ae.IntegStructure: IntegBuilder,
+        ae.DelayStructure: lambda x, y: DelayBuilder("Delay", x, y),
+        ae.DelayNStructure: lambda x, y: DelayBuilder("DelayN", x, y),
+        ae.DelayFixedStructure: DelayFixedBuilder,
+        ae.SmoothStructure: SmoothBuilder,
+        ae.SmoothNStructure: SmoothBuilder,
+        ae.TrendStructure: TrendBuilder,
+        ae.ForecastStructure: ForecastBuilder,
+        ae.SampleIfTrueStructure: SampleIfTrueBuilder,
+        ae.GetConstantsStructure: ExtConstantBuilder,
+        ae.GetDataStructure: ExtDataBuilder,
+        ae.GetLookupsStructure: ExtLookupBuilder,
+        ae.LookupsStructure: LookupsBuilder,
+        ae.InlineLookupsStructure: InlineLookupsBuilder,
+        ae.DataStructure: TabDataBuilder,
+        ae.ReferenceStructure: ReferenceBuilder,
+        ae.CallStructure: CallBuilder,
+        ae.GameStructure: GameBuilder,
+        ae.LogicStructure: OperationBuilder,
+        ae.ArithmeticStructure: OperationBuilder,
         int: NumericBuilder,
         float: NumericBuilder,
         np.ndarray: ArrayBuilder
@@ -1217,10 +1223,8 @@ def __init__(self, component):
         self.ast = component.ast
         self.subscripts = component.subscripts_dict
         self.component = component
-        # TODO add a attribute for "new structures"
 
     def visit(self):
-        # TODO: if final_subscripts == self.subscripts OK, else -> redimension
         visit_out = self._visit(self.ast)
 
         if not visit_out:
@@ -1239,6 +1243,8 @@ def visit(self):
 
         if not visit_out.subscripts\
           and self.subscripts != self.component.element.subs_dict:
+            # expression is a float, but it will be upper dimensioned
+            # when assigning values to the xarray.DataArray
             return visit_out
 
         # NUMPY not needed
@@ -1274,5 +1280,8 @@ def visit(self):
 
     def _visit(self, ast_object):
         builder = self.builders[type(ast_object)](ast_object, self.component)
-        arguments = {name: self._visit(value) for name, value in builder.arguments.items()}
+        arguments = {
+            name: self._visit(value)
+            for name, value in builder.arguments.items()
+        }
         return builder.build(arguments)

pysd/building/python/python_builder.py → pysd/building/python/python_model_builder.py

@@ -5,7 +5,7 @@
 from pysd.translation.structures.abstract_model import\
     AbstractComponent, AbstractElement, AbstractModel, AbstractSection
 
-from . import visitors as vs
+from . import python_expressions_builder as vs
 from .namespace import NamespaceManager
 from .subscripts import SubscriptManager
 from .imports import ImportsManager

pysd/pysd.py

@@ -23,14 +23,14 @@
     )
 
 
-def read_xmile(xmile_file, data_files=None, initialize=True,
+def read_xmile(xmile_file, data_files=None, initialize=True, old=False,
                missing_values="warning"):
     """
     Construct a model from `.xmile` file.
 
     Parameters
     ----------
-    xmile_file : str
+    xmile_file:  str or pathlib.Path
         The relative path filename for a raw `.xmile` file.
 
     initialize: bool (optional)
@@ -60,11 +60,20 @@ def read_xmile(xmile_file, data_files=None, initialize=True,
     >>> model = read_xmile('../tests/test-models/samples/teacup/teacup.xmile')
 
     """
-    from .translation.xmile.xmile2py import translate_xmile
+    if old:
+        # TODO: remove when this branch is ready to merge
+        from .translation.xmile.xmile2py import translate_xmile
+        py_model_file = translate_xmile(xmile_file)
+    else:
+        from pysd.translation.xmile.xmile_file import XmileFile
+        from pysd.building.python.python_model_builder import ModelBuilder
+        xmile_file_obj = XmileFile(xmile_file)
+        xmile_file_obj.parse()
 
-    py_model_file = translate_xmile(xmile_file)
+        abs_model = xmile_file_obj.get_abstract_model()
+        py_model_file = ModelBuilder(abs_model).build_model()
     model = load(py_model_file, data_files, initialize, missing_values)
-    model.xmile_file = xmile_file
+    model.xmile_file = str(xmile_file)
     return model
 
 
@@ -76,7 +85,7 @@ def read_vensim(mdl_file, data_files=None, initialize=True,
 
     Parameters
     ----------
-    mdl_file : str
+    mdl_file: str or pathlib.Path
         The relative path filename for a raw Vensim `.mdl` file.
 
     initialize: bool (optional)
@@ -87,7 +96,7 @@ def read_vensim(mdl_file, data_files=None, initialize=True,
         If given the list of files where the necessary data to run the model
         is given. Default is None.
 
-    missing_values : str ("warning", "error", "ignore", "keep") (optional)
+    missing_values: str ("warning", "error", "ignore", "keep") (optional)
         What to do with missing values. If "warning" (default)
         shows a warning message and interpolates the values.
         If "raise" raises an error. If "ignore" interpolates
@@ -126,11 +135,13 @@ def read_vensim(mdl_file, data_files=None, initialize=True,
 
     """
     if old:
+        # TODO: remove when this branch is ready to merge
         from .translation.vensim.vensim2py import translate_vensim
-        py_model_file = translate_vensim(mdl_file, split_views, encoding, **kwargs)
+        py_model_file = translate_vensim(
+            mdl_file, split_views, encoding, **kwargs)
     else:
-        from pysd.translation.vensim.vensin_file import VensimFile
-        from pysd.building.python.python_builder import ModelBuilder
+        from pysd.translation.vensim.vensim_file import VensimFile
+        from pysd.building.python.python_model_builder import ModelBuilder
         ven_file = VensimFile(mdl_file)
         ven_file.parse()
         if split_views:

pysd/tools/benchmarking.py

@@ -65,7 +65,7 @@ def runner(model_file, canonical_file=None, transpose=False, data_files=None,
     if model_file.suffix.lower() == ".mdl":
         model = read_vensim(model_file, data_files, old=old)
     elif model_file.suffix.lower() == ".xmile":
-        model = read_xmile(model_file, data_files)
+        model = read_xmile(model_file, data_files, old=old)
     elif model_file.suffix.lower() == ".py":
         model = load(model_file, data_files)
     else:

pysd/translation/vensim/vensim_element.py

@@ -3,10 +3,12 @@
 import parsimonious
 import numpy as np
 
-from ..structures.abstract_model import AbstractData, AbstractLookup, AbstractComponent, AbstractUnchangeableConstant
+from ..structures.abstract_model import\
+    AbstractData, AbstractLookup, AbstractComponent,\
+    AbstractUnchangeableConstant
 
 from . import vensim_utils as vu
-from .vensim_structures import structures, operators, parsing_ops
+from .vensim_structures import structures, parsing_ops
 
 
 class Element():  # File section dataclass

pysd/translation/vensim/vensin_file.py → pysd/translation/vensim/vensim_file.py

@@ -87,7 +87,7 @@ def parse(self):
             section.path = self.mdl_path.parent.joinpath(
                 self.clean_file_names(section.name)[0]
                 ).with_suffix(".py")
-        # TODO modify names and paths of macros
+
         for section in self.sections:
             section._parse()
 

pysd/translation/vensim/vensim_section.py

@@ -100,8 +100,6 @@ def merge_components(self):
 
             merged[name].components.append(component.get_abstract_component())
 
-
-
         return list(merged.values())
 
 

pysd/translation/vensim/vensim_structures.py

@@ -1,39 +1,39 @@
 import re
-from ..structures import components as cs
+from ..structures import abstract_expressions as ae
 
 
 structures = {
-    "reference":  cs.ReferenceStructure,
-    "subscripts_ref": cs.SubscriptsReferenceStructure,
-    "arithmetic": cs.ArithmeticStructure,
-    "logic": cs.LogicStructure,
-    "with_lookup": cs.InlineLookupsStructure,
-    "call": cs.CallStructure,
-    "game": cs.GameStructure,
-    "get_xls_lookups": cs.GetLookupsStructure,
-    "get_direct_lookups": cs.GetLookupsStructure,
-    "get_xls_data": cs.GetDataStructure,
-    "get_direct_data": cs.GetDataStructure,
-    "get_xls_constants": cs.GetConstantsStructure,
-    "get_direct_constants": cs.GetConstantsStructure,
-    "initial": cs.InitialStructure,
-    "integ": cs.IntegStructure,
-    "delay1": lambda x, y: cs.DelayStructure(x, y, x, 1),
-    "delay1i": lambda x, y, z: cs.DelayStructure(x, y, z, 1),
-    "delay3": lambda x, y: cs.DelayStructure(x, y, x, 3),
-    "delay3i": lambda x, y, z: cs.DelayStructure(x, y, z, 3),
-    "delay_n": cs.DelayNStructure,
-    "delay_fixed": cs.DelayFixedStructure,
-    "smooth": lambda x, y: cs.SmoothStructure(x, y, x, 1),
-    "smoothi": lambda x, y, z: cs.SmoothStructure(x, y, z, 1),
-    "smooth3": lambda x, y: cs.SmoothStructure(x, y, x, 3),
-    "smooth3i": lambda x, y, z: cs.SmoothStructure(x, y, z, 3),
-    "smooth_n": cs.SmoothNStructure,
-    "trend": cs.TrendStructure,
-    "forecast": cs.ForecastStructure,
-    "sample_if_true": cs.SampleIfTrueStructure,
-    "lookup": cs.LookupsStructure,
-    "data": cs.DataStructure
+    "reference":  ae.ReferenceStructure,
+    "subscripts_ref": ae.SubscriptsReferenceStructure,
+    "arithmetic": ae.ArithmeticStructure,
+    "logic": ae.LogicStructure,
+    "with_lookup": ae.InlineLookupsStructure,
+    "call": ae.CallStructure,
+    "game": ae.GameStructure,
+    "get_xls_lookups": ae.GetLookupsStructure,
+    "get_direct_lookups": ae.GetLookupsStructure,
+    "get_xls_data": ae.GetDataStructure,
+    "get_direct_data": ae.GetDataStructure,
+    "get_xls_constants": ae.GetConstantsStructure,
+    "get_direct_constants": ae.GetConstantsStructure,
+    "initial": ae.InitialStructure,
+    "integ": ae.IntegStructure,
+    "delay1": lambda x, y: ae.DelayStructure(x, y, x, 1),
+    "delay1i": lambda x, y, z: ae.DelayStructure(x, y, z, 1),
+    "delay3": lambda x, y: ae.DelayStructure(x, y, x, 3),
+    "delay3i": lambda x, y, z: ae.DelayStructure(x, y, z, 3),
+    "delay_n": ae.DelayNStructure,
+    "delay_fixed": ae.DelayFixedStructure,
+    "smooth": lambda x, y: ae.SmoothStructure(x, y, x, 1),
+    "smoothi": lambda x, y, z: ae.SmoothStructure(x, y, z, 1),
+    "smooth3": lambda x, y: ae.SmoothStructure(x, y, x, 3),
+    "smooth3i": lambda x, y, z: ae.SmoothStructure(x, y, z, 3),
+    "smooth_n": ae.SmoothNStructure,
+    "trend": ae.TrendStructure,
+    "forecast": ae.ForecastStructure,
+    "sample_if_true": ae.SampleIfTrueStructure,
+    "lookup": ae.LookupsStructure,
+    "data": ae.DataStructure
 }
 
 

pysd/translation/vensim/vensim_utils.py

@@ -10,7 +10,7 @@
 
 class Grammar():
     _common_grammar = None
-    _grammar_path: Path = Path(__file__).parent.joinpath("parsing_expr")
+    _grammar_path: Path = Path(__file__).parent.joinpath("parsing_grammars")
     _grammar: Dict = {}
 
     @classmethod