188 discrepancies betwee import and export formats rebased

.travis.yml

@@ -12,9 +12,10 @@ before_install:
   - cp elixir_daisy/settings_compose_ci.py elixir_daisy/settings_compose.py
 
 install:
+  - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
   - docker-compose build
   - docker-compose up -d
   - sleep 60
 
 script:
-  - docker-compose exec web python setup.py pytest 
+  - docker-compose exec web python setup.py pytest 

README.md

@@ -93,21 +93,19 @@ You are encouraged to try Daisy for yourself using our [DEMO deployment](https:/
 
 #### Importing 
 
-In addition to loading of initial data, DAISY database can be populated by importing Project and Dataset records from JSON files.
- The commands for import are given below: </br>
+In addition to loading of initial data, DAISY database can be populated by importing Project, Dataset and Partners records from JSON files using commands `import_projects`, `import_datasets` and `import_partners` respectively.
+ The commands for import are accepting one JSON file (flag `-f`): </br>
 
 ```bash
-docker-compose exec web python manage.py import_projects -f ${PROJECTS_JSON}
+docker-compose exec web python manage.py <COMMAND> -f ${PATH_TO_JSON_FILE}
 ```
-where ${PROJECTS_JSON} is the path to a json file containing the projects definitions.  
-See file daisy/data/projects.json as an example.
+where ${PATH_TO_JSON_FILE} is the path to a json file containing the records definitions.
+See file daisy/data/demo/projects.json as an example.
 
-
+Alternatively, you can specify directory containing multiple JSON files to be imported with `-d` flag:
 ```bash
-docker-compose exec web python manage.py import_datasets -d ${DATASETS_FOLDER}
+docker-compose exec web python manage.py <COMMAND> -d ${PATH_TO_DIR}
 ```
-where ${DATASETS_FOLDER} is the path to a folder containing datasets and data declarations definitions.  
-See folder daisy/data/datasets as an example.
 
 #### Exporting  
 
@@ -206,16 +204,21 @@ To be completed.
 ./manage.py import_users
 ```
 
-### Import projects from external system
+### Import projects, datasets or partners from external system
+Single file mode:
 ```bash
 ./manage.py import_projects -f path/to/json_file.json
 ```
 
-### Import datasets from external system
+Batch mode:
 ```bash
-./manage.py import_datasets -d path/to/folder_with_json
+./manage.py import_projects -f path/to/dir/with/json/files/
 ```
 
+Available commands: `import_projects`, `import_datasets`, `import_partners`.
+
+In case of problems, add `--verbose` flag to the command, and take a look inside `./log/daisy.log`. 
+
 ### Install js and css dependencies
 
 ```bash

conftest.py

@@ -179,6 +179,14 @@ def storage_resources():
     CommandLoadInitialData.create_storage_resources()
 
 
+@pytest.fixture
+def data_types():
+    """
+    Create data types
+    """
+    CommandLoadInitialData.create_datatypes()
+
+
 @pytest.fixture
 def partners():
     """

core/exceptions.py

@@ -1,7 +1,7 @@
 class DaisyError(Exception):
     """Base error class for the module."""
 
-    default_message = "Unspecified daisy error"
+    default_message = "Unspecified DAISY error"
 
     def __init__(self, msg=None):
         self._msg = msg or self.default_message
@@ -37,3 +37,13 @@ class FixtureImportError(DaisyError):
     def __init__(self, data, msg=None):
         self.data = data
         super().__init__(msg=msg)
+
+
+class JSONSchemaValidationError(DaisyError):
+    """Error when validating data to be imported."""
+
+    default_message = "Error when validating data against JSON schema:' {data}'"
+
+    def __init__(self, data, msg=None):
+        self.data = data
+        super().__init__(msg=msg)

core/fixtures/elu-dataset.json

@@ -0,0 +1,285 @@
+{
+  "description": "ELIXIR Luxembourg Projects Schema",
+  "title": "A JSON Schema for describing biomedical research Projects.",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "allOf": [
+    {
+      "$ref": "https://git-r3lab.uni.lu/pinar.alper/metadata-tools/raw/master/metadata_tools/resources/elu-core.json"
+    }
+  ],
+  "properties": {
+    "project": {
+      "description": "Name of the project the dataset belongs to.",
+      "type": ["string", "null"]
+    },
+    "data_declarations": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/elu:data"
+      }
+    },
+    "storages": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/elu:storage"
+      }
+    },
+    "legal_bases": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/elu:legal_basis"
+      }
+    },
+    "transfers": {
+      "type": "array",
+      "items": {
+        "$ref": "#/definitions/elu:transfer"
+      }
+    }
+  },
+  "definitions": {
+    "elu:storage": {
+      "type": "object",
+      "properties": {
+        "platform": {"type": "string"},
+        "location": {"type": "string"},
+        "accesses": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      }
+    },
+    "elu:transfer": {
+      "type": "object",
+      "properties": {
+        "partner": {"type": "string"},
+        "transfer_details": {"type": ["string", "null"]},
+        "transfer_date": {
+          "type": ["string", "null"],
+          "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+        }
+      }
+    },
+    "elu:legal_basis": {
+      "type": "object",
+      "properties": {
+        "data_declarations": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "legal_basis_codes": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "enum": [
+              "6.1(a)",
+              "6.1(b)",
+              "6.1(c)",
+              "6.1(d)",
+              "6.1(e)",
+              "6.1(f)"
+            ]
+          }},
+        "personal_data_codes": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "enum": [
+              "Standard",
+              "Special",
+              "Race",
+              "Genetic",
+              "Biometric",
+              "Health",
+              "Sexual orientation",
+              "Convictions",
+              "Other"
+            ]
+          }
+        },
+        "legal_basis_notes": {
+          "type": "string"
+        }
+      },
+      "required": [
+        "legal_basis_codes",
+        "personal_data_codes"
+      ]
+    },
+    "elu:data": {
+      "type": "object",
+      "properties": {
+        "title": {
+          "type": "string"
+        },
+        "use_restrictions": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "use_class": {
+                "type": "string",
+                "enum": [
+                  "NRES",
+                  "GRU(CC)",
+                  "HMB(CC)",
+                  "DS-[XX](CC)",
+                  "POA",
+                  "RS-[XX]",
+                  "RUO",
+                  "NMDS",
+                  "GSO",
+                  "NPU",
+                  "PUB",
+                  "COL-[XX]",
+                  "RTN",
+                  "IRB",
+                  "GS-[XX]",
+                  "MOR-[XX]",
+                  "TS-[XX]",
+                  "US",
+                  "PS",
+                  "IS",
+                  "Other"
+                ]
+              },
+              "use_class_note": {
+                "type": "string"
+              },
+              "use_restriction_rule": {
+                "type": "string",
+                "enum": [
+                  "CONSTRAINTS",
+                  "NO_CONSTRAINTS",
+                  "FORBIDDEN"
+                ]
+              },
+              "use_restriction_objects": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                },
+                "use_restriction_note": {
+                  "type": "string"
+                }
+              }
+            },
+            "required": [
+              "use_class_note",
+              "use_restriction_rule"
+            ]
+          }
+        },
+        "data_types": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "enum": [
+              "ATAC-seq",
+              "Biometric_data",
+              "ChIP-seq",
+              "Samples",
+              "Disease status",
+              "Ethnic_origin",
+              "Imaging",
+              "Omics_data",
+              "Genotype_data",
+              "Whole_genome_sequencing",
+              "Exome_sequencing",
+              "Genomics_variant_array",
+              "RNASeq",
+              "Genetic_and_derived_genetic_data",
+              "Transcriptome_array",
+              "Methylation_array",
+              "MicroRNA_array",
+              "Metabolomics",
+              "Metagenomics",
+              "Metatranscriptomics",
+              "Metaproteomics",
+              "Single_Cell_RNAseq",
+              "Proteomics",
+              "Other_omics_data",
+              "Clinical_Imaging",
+              "Cell_Imaging",
+              "Human_subject_data",
+              "Clinical_data",
+              "Lifestyle_data",
+              "Socio_Economic_Data",
+              "Environmental_Data",
+              "Other_Phenotype_data",
+              "Other"
+            ]
+          },
+          "data_type_notes": {
+            "type": "string"
+          },
+          "access_category": {
+            "type": "string",
+            "enum": [
+              "open-access",
+              "controlled-access",
+              "registered-access"
+            ]
+          },
+          "subjects_category": {
+            "type": "array",
+            "items": {
+              "type": "string",
+              "enum": [
+                "cases",
+                "controls",
+                "cases_and_controls"
+              ]
+            }
+          },
+          "de_identification": {
+            "type": "string",
+            "enum": [
+              "anonymized",
+              "pseudonymized"
+            ]
+          },
+          "consent_status": {
+            "type": "string",
+            "enum": [
+              "heterogeneous",
+              "homogeneous"
+            ]
+          },
+          "consent_status_description": {
+            "type": "string"
+          },
+          "has_special_subjects": {
+            "type": "boolean"
+          },
+          "special_subjects_description": {
+            "type": "string"
+          },
+          "embargo_date": {
+            "type": "string",
+            "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+          },
+          "storage_end_date": {
+            "type": "string",
+            "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+          },
+          "storage_duration_criteria": {
+            "type": "string",
+            "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"
+          }
+        }
+      }
+    }
+  }
+}
+
+
+
+
+
+