Amend json schema and load dataset from structure function.

Implemented an InputValidationException with its message
Meaningful-Data · Jan 7, 2025 · d7cde62 · d7cde62
1 parent 7b2e595
commit d7cde62
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 16 deletions.
diff --git a/src/vtlengine/API/_InternalApi.py b/src/vtlengine/API/_InternalApi.py
@@ -41,11 +41,19 @@ def _load_dataset_from_structure(structures: Dict[str, Any]) -> Dict[str, Any]:
                 try:
                     validate(instance=dataset_json["components"], schema=schema)
                 except exceptions.ValidationError as e:
-                    raise InputValidationException(f" 0- {e.message}")
+                    raise InputValidationException(code="0-3-1-1", message=e.message)
 
                 for component in dataset_json["components"]:
                     check_key("data_type", SCALAR_TYPES.keys(), component["data_type"])
                     check_key("role", Role_keys, component["role"])
+                    if "nullable" not in dataset_json["components"]:
+                        if Role(component["role"]) == Role.IDENTIFIER:
+                            component["nullable"] = False
+                        elif Role(component["role"]) == Role.MEASURE:
+                            component["nullable"] = True
+                        else:
+                            component["nullable"] = False
+
                     components[component["name"]] = Component(
                         name=component["name"],
                         data_type=SCALAR_TYPES[component["data_type"]],

diff --git a/src/vtlengine/Exceptions/messages.py b/src/vtlengine/Exceptions/messages.py
@@ -17,6 +17,7 @@
     "0-1-2-5": "File {file} must be encoded in utf-8 (without BOM).",
     # "0-1-2-5": "The library item {li}, used in this module {mdl}, is not found.",
     # JSON Schema validations
+    "0-3-1-1": "Dataset {dataset} is not valid according to JSON schema",
     # Infer Data Structure errors
     # "0-1-1-1": "A csv file or a dataframe is required.",
     "0-1-1-2": "The provided {source} must have data to can infer the data structure.",

diff --git a/tests/API/data/DataStructure/input/DS_1.json b/tests/API/data/DataStructure/input/DS_1.json
@@ -2,24 +2,31 @@
   "datasets": [
     {
       "name": "DS_1",
-      "DataStructure": [
+      "components": [
         {
           "name": "Id_1",
           "role": "Identifier",
-          "type": "Integer",
-          "nullable": false
+          "data_type": "Time_Period"
         },
         {
           "name": "Id_2",
           "role": "Identifier",
-          "type": "String",
-          "nullable": false
+          "data_type": "String"
+        },
+        {
+          "name": "Id_3",
+          "role": "Identifier",
+          "data_type": "String"
         },
         {
           "name": "Me_1",
           "role": "Measure",
-          "type": "Number",
-          "nullable": true
+          "data_type": "Integer"
+        },
+        {
+          "name": "At_1",
+          "role": "Attribute",
+          "data_type": "String"
         }
       ]
     }

diff --git a/tests/API/data/DataStructure/input/DS_Schema.json b/tests/API/data/DataStructure/input/DS_Schema.json
@@ -6,20 +6,22 @@
         {
           "name": "Id_1",
           "role": "Identifier",
-          "data_type": "Integer",
-          "nullable": false
+          "data_type": "String"
         },
         {
           "name": "Id_2",
           "role": "Identifier",
-          "data_type": "String",
-          "nullable": false
+          "data_type": "String"
         },
         {
           "name": "Me_1",
           "role": "Measure",
-          "data_type": "Number",
-          "nullable": true
+          "data_type": "Integer"
+        },
+        {
+          "name": "At_1",
+          "role": "Attribute",
+          "data_type": "String"
         }
       ]
     }

diff --git a/tests/API/test_api.py b/tests/API/test_api.py
@@ -972,7 +972,7 @@ def test_load_data_structure_with_new_schema(data_structure):
         components={
             "Id_1": Component(
                 name="Id_1",
-                data_type=DataTypes.Integer,
+                data_type=DataTypes.String,
                 role=Role.IDENTIFIER,
                 nullable=False,
             ),
@@ -984,10 +984,16 @@ def test_load_data_structure_with_new_schema(data_structure):
             ),
             "Me_1": Component(
                 name="Me_1",
-                data_type=DataTypes.Number,
+                data_type=DataTypes.Integer,
                 role=Role.MEASURE,
                 nullable=True,
             ),
+            "At_1": Component(
+                name="At_1",
+                data_type=DataTypes.String,
+                role=Role.ATTRIBUTE,
+                nullable=False,
+            ),
         },
         data=None,
     )