45 check time period comparison (#46)

* Fixed Time_Period compraison (It just compare the period instead of the date itself). Refactored all time period tests to work with pytest. Added some basic Time_Period comparison tests: =, <>, <, >, <=, >=.

* Fixed Time Period Comparison tests.

* Fixed mypy and ruff errors.

* Fixed ruff error.

* Merged main into branch.

* Refactored timePeriod meta comparison. Updated tests to work with the new comparison criteria.

* meta_comparison optimized.

* Fixed ruff errors.

* Fixed mypy errors.

* Ruff format launched.

* Ruff format launched.

* Ruff format launched.

* Fixed last version mypy errors.

* Updated pyproject: mypy and ruff versions.

---------

Signed-off-by: Mateo de Lorenzo Argelés <160473799+mla2001@users.noreply.github.com>
Co-authored-by: Francisco Javier Hernández del Caño <javier.hernandez@meaningfuldata.eu>

pyproject.toml

@@ -45,7 +45,7 @@ pytest-cov = "^5.0.0"
 line-profiler-pycharm = "^1.2.0"
 sphinx = "^7.4"
 sphinx-rtd-theme = "^2.0.0"
-mypy = "^1.11.2"
+mypy = "^1.14.1"
 pandas-stubs = "^2.2.3.241009"
 stubs = "^1.0.0"
 toml = "^0.10.2"

src/vtlengine/AST/ASTConstructorModules/Expr.py

@@ -828,26 +828,6 @@ def visitTimeUnaryAtom(self, ctx: Any):
 
         return UnaryOp(op=op, operand=operand_node[0])
 
-    # def visitPeriodAtom(self, ctx: Parser.PeriodAtomContext):
-    #     """
-    #     periodExpr: PERIOD_INDICATOR '(' expr? ')' ;
-    #     """
-    #     ctx_list = list(ctx.getChildren())
-    #     c = ctx_list[0]
-    #
-    #     op = c.getSymbol().text
-    #     operand_node = [
-    #         self.visitExpr(operand)
-    #         for operand in ctx_list
-    #         if isinstance(operand, Parser.ExprContext)
-    #     ]
-    #
-    #     if len(operand_node) == 0:
-    #         # AST_ASTCONSTRUCTOR.15
-    #         raise NotImplementedError
-    #
-    #     return UnaryOp(op=op, operand=operand_node[0])
-
     def visitTimeShiftAtom(self, ctx: Parser.TimeShiftAtomContext):
         """
         timeShiftExpr: TIMESHIFT '(' expr ',' INTEGER_CONSTANT ')' ;

src/vtlengine/DataTypes/TimeHandling.py

@@ -203,6 +203,10 @@ def period_indicator(self, value: str) -> None:
             raise SemanticError("2-1-19-2", period=value)
         self._period_indicator = value
 
+    @property
+    def period_magnitude(self) -> int:
+        return DURATION_MAPPING[self.period_indicator]
+
     @property
     def period_number(self) -> int:
         return self._period_number
@@ -230,17 +234,46 @@ def period_number(self, value: int) -> None:
                     # raise ValueError(f'Invalid day {value} for year {self.year}.')
         self._period_number = value
 
+    @property
+    def period_dates(self) -> tuple[date, date]:
+        return (
+            period_to_date(self.year, self.period_indicator, self.period_number, start=True),
+            period_to_date(self.year, self.period_indicator, self.period_number, start=False),
+        )
+
     def _meta_comparison(self, other: Any, py_op: Any) -> Optional[bool]:
         if pd.isnull(other):
             return None
+
+        if py_op in (operator.eq, operator.ne):
+            return py_op(str(self), str(other))
+
+        if py_op in (operator.ge, operator.le) and str(self) == str(other):
+            return True
+
         if isinstance(other, str):
-            if len(other) == 0:
-                return False
             other = TimePeriodHandler(other)
-        return py_op(
-            DURATION_MAPPING[self.period_indicator],
-            DURATION_MAPPING[other.period_indicator],
-        )
+
+        self_lapse, other_lapse = self.period_dates, other.period_dates
+        is_lt_or_le = py_op in [operator.lt, operator.le]
+        is_gt_or_ge = py_op in [operator.gt, operator.ge]
+
+        if is_lt_or_le or is_gt_or_ge:
+            idx = 0 if is_lt_or_le else 1
+            if self_lapse[idx] != other_lapse[idx]:
+                return (
+                    self_lapse[idx] < other_lapse[idx]
+                    if is_lt_or_le
+                    else self_lapse[idx] > other_lapse[idx]
+                )
+            if self.period_magnitude != other.period_magnitude:
+                return (
+                    self.period_magnitude < other.period_magnitude
+                    if is_lt_or_le
+                    else self.period_magnitude > other.period_magnitude
+                )
+
+        return False
 
     def start_date(self, as_date: bool = False) -> Union[date, str]:
         """

src/vtlengine/Operators/Conditional.py

@@ -66,6 +66,7 @@ def component_level_evaluation(
             else:
                 false_data = false_branch.data.reindex(condition.data.index)
             result = np.where(condition.data, true_data, false_data)
+
         return pd.Series(result, index=condition.data.index)  # type: ignore[union-attr]
 
     @classmethod

src/vtlengine/Operators/HROperators.py

@@ -26,9 +26,7 @@ def get_measure_from_dataset(dataset: Dataset, code_item: str) -> DataComponent:
 class HRComparison(Operators.Binary):
     @classmethod
     def imbalance_func(cls, x: Any, y: Any) -> Any:
-        if pd.isnull(x) or pd.isnull(y):
-            return None
-        return x - y
+        return None if pd.isnull(x) or pd.isnull(y) else x - y
 
     @staticmethod
     def hr_func(left_series: Any, right_series: Any, hr_mode: str) -> Any:

tests/Attributes/data/DataSet/output/5-4-1-27-1.csv

@@ -1,5 +1,5 @@
 Id_1,Id_2,Id_3,Id_4,bool_var
-2012,B,Total,Total,True
+2012,B,Total,Total,False
 2012,G,Total,Total,True
 2012,S,Total,Total,True
 2012,M,Total,Total,True

tests/Attributes/data/DataSet/output/5-4-1-28-1.csv

@@ -1,7 +1,7 @@
 Id_1,Id_2,Id_3,Id_4,Me_1,Me_2,At_1,At_2,Me_3
 2012,B,Total,Total,2010Q3,2010Q3,Dolar,Euro,True
-2012,G,Total,Total,2010Q1,2010Q3,Dolar,Euro,True
-2012,S,Total,Total,2010Q1,2010Q3,Dolar,Euro,True
-2012,M,Total,Total,2010Q1,2010Q3,Dolar,Euro,True
-2012,F,Total,Total,2010Q1,2010Q3,Dolar,Euro,True
-2012,W,Total,Total,2010Q1,2010Q3,Dolar,Euro,True
+2012,G,Total,Total,2010Q1,2010Q3,Dolar,Euro,False
+2012,S,Total,Total,2010Q1,2010Q3,Dolar,Euro,False
+2012,M,Total,Total,2010Q1,2010Q3,Dolar,Euro,False
+2012,F,Total,Total,2010Q1,2010Q3,Dolar,Euro,False
+2012,W,Total,Total,2010Q1,2010Q3,Dolar,Euro,False

tests/Bugs/data/vtl/GL_195_1.vtl

@@ -1 +1 @@
-B03.ANAMART_INSTRMNT_SHR_ONA_CRDTR := B03.INSTRMNT_FCT_TRNSFRRD_AMNT [ calc SHR_ONA_CRDTR := if OTSTNDNG_NMNL_AMNT = 0 then 1 / NMBR_CRDTR else ( if OBSRVD_AGNT_CD = CRDTR_ID then ( OTSTNDNG_NMNL_AMNT - TRNSFRRD_AMNT ) / OTSTNDNG_NMNL_AMNT else ( if OTSTNDNG_NMNL_AMNT * NMBR_CRDTRS_NT_OA = 0.0 then 1 else TRNSFRRD_AMNT / ( OTSTNDNG_NMNL_AMNT * NMBR_CRDTRS_NT_OA ) ) ) ] [ keep SHR_ONA_CRDTR ] ; 
+B03.ANAMART_INSTRMNT_SHR_ONA_CRDTR := B03.INSTRMNT_FCT_TRNSFRRD_AMNT [ calc SHR_ONA_CRDTR := if OTSTNDNG_NMNL_AMNT = 0 then 1 / NMBR_CRDTR else ( if OBSRVD_AGNT_CD = CRDTR_ID then ( OTSTNDNG_NMNL_AMNT - TRNSFRRD_AMNT ) / OTSTNDNG_NMNL_AMNT else ( if OTSTNDNG_NMNL_AMNT * NMBR_CRDTRS_NT_OA = 0.0 then 1 else TRNSFRRD_AMNT / ( OTSTNDNG_NMNL_AMNT * NMBR_CRDTRS_NT_OA ) ) ) ] [ keep SHR_ONA_CRDTR ] ;

tests/TimePeriod/conftest.py

@@ -0,0 +1,48 @@
+import json
+import os
+
+import pandas as pd
+import pytest
+
+from vtlengine.API._InternalApi import load_datasets_with_data
+from vtlengine.Exceptions import SemanticError
+
+
+def load_datasets(base_path, code, folder_type):
+    datapoints_path = base_path / "DataSet" / folder_type
+    input_path = base_path / "DataStructure" / folder_type
+
+    num_inputs = len([f for f in os.listdir(input_path) if f.startswith(f"{code}-")])
+    datasets = {}
+
+    for i in range(1, num_inputs + 1):
+        with open(input_path / f"{code}-{i}.json", "r") as file:
+            datastructure = json.load(file)
+        if "datasets" in datastructure:
+            ds_name = datastructure["datasets"][0]["name"]
+            datapoint = {ds_name: pd.read_csv(datapoints_path / f"{code}-{i}.csv")}
+        else:
+            datapoint = None
+        datasets.update(load_datasets_with_data(datastructure, datapoint)[0])
+
+    return datasets
+
+
+@pytest.fixture
+def load_input(request, code):
+    base_path = request.node.get_closest_marker("input_path").args[0]
+    return load_datasets(base_path, code, folder_type="input")
+
+
+@pytest.fixture
+def load_reference(request, code):
+    base_path = request.node.get_closest_marker("input_path").args[0]
+    return load_datasets(base_path, code, folder_type="output")
+
+
+@pytest.fixture
+def load_error(request, code):
+    with pytest.raises(SemanticError) as context:
+        base_path = request.node.get_closest_marker("input_path").args[0]
+        return load_datasets(base_path, code, folder_type="input")
+    return context.value.args[1]

tests/TimePeriod/data/DataSet/input/1-1.csv

@@ -0,0 +1,49 @@
+CONF_STATUS,EXR_CURRENCY,FREQ,OBS_STATUS,OBS_VALUE,TIME_PERIOD
+,01,M,,112,2000M1
+,02,M,,212,2000M2
+,03,M,A,312,2000M3
+,04,M,A,412,2000M4
+,05,M,A,512,2000M5
+,CNY,M,A,612,2000M6
+,CNY,M,A,712,2000M7
+,CNY,M,A,812,2000M8
+,CNY,M,A,912,2000M9
+,CNY,M,A,1120,2000M10
+,CNY,M,A,1121,2000M11
+,CNY,M,A,1122,2000M12
+,CNY,M,A,112,2001M1
+,CNY,M,A,212,2001M2
+,CNY,M,A,312,2001M3
+,CNY,M,A,412,2001M4
+,CNY,M,A,512,2001M5
+,CNY,M,A,612,2001M6
+,CNY,M,A,712,2001M7
+,CNY,M,A,812,2001M8
+,CNY,M,A,912,2001M9
+,CNY,M,A,1120,2001M10
+,CNY,M,A,1121,2001M11
+,CNY,M,A,1122,2001M12
+,CNY,M,A,112,2002M1
+,CNY,M,A,212,2002M2
+,CNY,M,A,312,2002M3
+,CNY,M,A,412,2002M4
+,CNY,M,A,512,2002M5
+,CNY,M,A,612,2002M6
+,CNY,M,A,712,2002M7
+,CNY,M,A,812,2002M8
+,CNY,M,A,912,2002M9
+,CNY,M,A,1120,2002M10
+,CNY,M,A,1121,2002M11
+,CNY,M,A,1122,2002M12
+,CNY,M,A,112,2003M1
+,CNY,M,A,212,2003M2
+,CNY,M,A,312,2003M3
+,CNY,M,A,412,2003M4
+,CNY,M,A,512,2003M5
+,CNY,M,A,612,2003M6
+,CNY,M,A,712,2003M7
+,CNY,M,A,812,2003M8
+,CNY,M,A,912,2003M9
+,CNY,M,A,1120,2003M10
+,CNY,M,A,1121,2003M11
+,CNY,M,A,1122,2003M12

tests/TimePeriod/data/DataSet/input/2-1.csv

@@ -0,0 +1,49 @@
+CONF_STATUS,EXR_CURRENCY,FREQ,OBS_STATUS,OBS_VALUE,TIME_PERIOD
+,01,M,,112,2000M1
+,02,M,,212,2000M2
+,03,M,A,312,2000M3
+,04,M,A,412,2000M4
+,05,M,A,512,2000M5
+,CNY,M,A,612,2000M6
+,CNY,M,A,712,2000M7
+,CNY,M,A,812,2000M8
+,CNY,M,A,912,2000M9
+,CNY,M,A,1120,2000M10
+,CNY,M,A,1121,2000M11
+,CNY,M,A,1122,2000M12
+,CNY,M,A,112,2001M1
+,CNY,M,A,212,2001M2
+,CNY,M,A,312,2001M3
+,CNY,M,A,412,2001M4
+,CNY,M,A,512,2001M5
+,CNY,M,A,612,2001M6
+,CNY,M,A,712,2001M7
+,CNY,M,A,812,2001M8
+,CNY,M,A,912,2001M9
+,CNY,M,A,1120,2001M10
+,CNY,M,A,1121,2001M11
+,CNY,M,A,1122,2001M12
+,CNY,M,A,112,2002M1
+,CNY,M,A,212,2002M2
+,CNY,M,A,312,2002M3
+,CNY,M,A,412,2002M4
+,CNY,M,A,512,2002M5
+,CNY,M,A,612,2002M6
+,CNY,M,A,712,2002M7
+,CNY,M,A,812,2002M8
+,CNY,M,A,912,2002M9
+,CNY,M,A,1120,2002M10
+,CNY,M,A,1121,2002M11
+,CNY,M,A,1122,2002M12
+,CNY,M,A,112,2003M1
+,CNY,M,A,212,2003M2
+,CNY,M,A,312,2003M3
+,CNY,M,A,412,2003M4
+,CNY,M,A,512,2003M5
+,CNY,M,A,612,2003M6
+,CNY,M,A,712,2003M7
+,CNY,M,A,812,2003M8
+,CNY,M,A,912,2003M9
+,CNY,M,A,1120,2003M10
+,CNY,M,A,1121,2003M11
+,CNY,M,A,1122,2003M12

tests/TimePeriod/data/DataSet/input/3-1.csv

@@ -0,0 +1,49 @@
+CONF_STATUS,EXR_CURRENCY,FREQ,OBS_STATUS,OBS_VALUE,TIME_PERIOD
+,01,M,,112,2000M1
+,02,M,,212,2000M2
+,03,M,A,312,2000M3
+,04,M,A,412,2000M4
+,05,M,A,512,2000M5
+,CNY,M,A,612,2000M6
+,CNY,M,A,712,2000M7
+,CNY,M,A,812,2000M8
+,CNY,M,A,912,2000M9
+,CNY,M,A,1120,2000M10
+,CNY,M,A,1121,2000M11
+,CNY,M,A,1122,2000M12
+,CNY,M,A,112,2001M1
+,CNY,M,A,212,2001M2
+,CNY,M,A,312,2001M3
+,CNY,M,A,412,2001M4
+,CNY,M,A,512,2001M5
+,CNY,M,A,612,2001M6
+,CNY,M,A,712,2001M7
+,CNY,M,A,812,2001M8
+,CNY,M,A,912,2001M9
+,CNY,M,A,1120,2001M10
+,CNY,M,A,1121,2001M11
+,CNY,M,A,1122,2001M12
+,CNY,M,A,112,2002M1
+,CNY,M,A,212,2002M2
+,CNY,M,A,312,2002M3
+,CNY,M,A,412,2002M4
+,CNY,M,A,512,2002M5
+,CNY,M,A,612,2002M6
+,CNY,M,A,712,2002M7
+,CNY,M,A,812,2002M8
+,CNY,M,A,912,2002M9
+,CNY,M,A,1120,2002M10
+,CNY,M,A,1121,2002M11
+,CNY,M,A,1122,2002M12
+,CNY,M,A,112,2003M1
+,CNY,M,A,212,2003M2
+,CNY,M,A,312,2003M3
+,CNY,M,A,412,2003M4
+,CNY,M,A,512,2003M5
+,CNY,M,A,612,2003M6
+,CNY,M,A,712,2003M7
+,CNY,M,A,812,2003M8
+,CNY,M,A,912,2003M9
+,CNY,M,A,1120,2003M10
+,CNY,M,A,1121,2003M11
+,CNY,M,A,1122,2003M12

tests/TimePeriod/data/DataSet/input/4-1.csv

@@ -0,0 +1,49 @@
+CONF_STATUS,EXR_CURRENCY,FREQ,OBS_STATUS,OBS_VALUE,TIME_PERIOD
+,01,M,,112,2000M1
+,02,M,,212,2000M2
+,03,M,A,312,2000M3
+,04,M,A,412,2000M4
+,05,M,A,512,2000M5
+,CNY,M,A,612,2000M6
+,CNY,M,A,712,2000M7
+,CNY,M,A,812,2000M8
+,CNY,M,A,912,2000M9
+,CNY,M,A,1120,2000M10
+,CNY,M,A,1121,2000M11
+,CNY,M,A,1122,2000M12
+,CNY,M,A,112,2001M1
+,CNY,M,A,212,2001M2
+,CNY,M,A,312,2001M3
+,CNY,M,A,412,2001M4
+,CNY,M,A,512,2001M5
+,CNY,M,A,612,2001M6
+,CNY,M,A,712,2001M7
+,CNY,M,A,812,2001M8
+,CNY,M,A,912,2001M9
+,CNY,M,A,1120,2001M10
+,CNY,M,A,1121,2001M11
+,CNY,M,A,1122,2001M12
+,CNY,M,A,112,2002M1
+,CNY,M,A,212,2002M2
+,CNY,M,A,312,2002M3
+,CNY,M,A,412,2002M4
+,CNY,M,A,512,2002M5
+,CNY,M,A,612,2002M6
+,CNY,M,A,712,2002M7
+,CNY,M,A,812,2002M8
+,CNY,M,A,912,2002M9
+,CNY,M,A,1120,2002M10
+,CNY,M,A,1121,2002M11
+,CNY,M,A,1122,2002M12
+,CNY,M,A,112,2003M1
+,CNY,M,A,212,2003M2
+,CNY,M,A,312,2003M3
+,CNY,M,A,412,2003M4
+,CNY,M,A,512,2003M5
+,CNY,M,A,612,2003M6
+,CNY,M,A,712,2003M7
+,CNY,M,A,812,2003M8
+,CNY,M,A,912,2003M9
+,CNY,M,A,1120,2003M10
+,CNY,M,A,1121,2003M11
+,CNY,M,A,1122,2003M12