Feature/pepys config file

config.py

@@ -0,0 +1,63 @@
+import os
+
+from configparser import ConfigParser
+
+from pepys_import.utils.config_utils import process
+
+config = ConfigParser(allow_no_value=True)
+DEFAULT_CONFIG_FILE_PATH = os.path.join(os.path.dirname(__file__), "default_config.ini")
+CONFIG_FILE_PATH = os.getenv("PEPYS_CONFIG_FILE", DEFAULT_CONFIG_FILE_PATH)
+
+if not os.path.exists(CONFIG_FILE_PATH):
+    raise Exception(f"No such file: '{CONFIG_FILE_PATH}'.")
+elif not os.path.isfile(CONFIG_FILE_PATH):
+    raise Exception(
+        f"Your environment variable doesn't point to a file: '{CONFIG_FILE_PATH}'."
+    )
+
+# Read the config file
+config.read(CONFIG_FILE_PATH)
+
+assert config.has_section(
+    "database"
+), f"'database' section couldn't find in '{CONFIG_FILE_PATH}'!"
+
+# Fetch database section
+DB_USERNAME = config.get("database", "db_username", fallback="postgres")
+DB_PASSWORD = config.get("database", "db_password", fallback="postgres")
+DB_HOST = config.get("database", "db_host", fallback="localhost")
+DB_PORT = config.getint("database", "db_port", fallback="5432")
+DB_NAME = config.get("database", "db_name", fallback="pepys")
+DB_TYPE = config.get("database", "db_type", fallback="postgres")
+
+# Process username and password if necessary
+if DB_USERNAME.startswith("_") and DB_USERNAME.endswith("_"):
+    DB_USERNAME = process(DB_USERNAME[1:-1])
+if DB_PASSWORD.startswith("_") and DB_PASSWORD.startswith("_"):
+    DB_PASSWORD = process(DB_PASSWORD[1:-1])
+
+assert config.has_section(
+    "archive"
+), f"'archive' section couldn't find in '{CONFIG_FILE_PATH}'!"
+
+# Fetch archive section
+# TODO: The following username and password might be necessary when files are tried to be moved to
+# the archive path
+ARCHIVE_USER = config.get("archive", "user")
+ARCHIVE_PASSWORD = config.get("archive", "password")
+ARCHIVE_PATH = config.get("archive", "path")
+
+# Process user and password if necessary
+if ARCHIVE_USER.startswith("_") and ARCHIVE_USER.endswith("_"):
+    ARCHIVE_USER = process(ARCHIVE_USER[1:-1])
+if ARCHIVE_PASSWORD.startswith("_") and ARCHIVE_PASSWORD.endswith("_"):
+    ARCHIVE_PASSWORD = process(ARCHIVE_PASSWORD[1:-1])
+
+assert config.has_section(
+    "local"
+), f"'local' section couldn't find in '{CONFIG_FILE_PATH}'!"
+
+# Fetch local section
+LOCAL_PARSERS = config.get("local", "parsers")
+LOCAL_BASIC_TESTS = config.get("local", "basic_tests")
+LOCAL_ENHANCED_TESTS = config.get("local", "enhanced_tests")

default_config.ini

@@ -0,0 +1,15 @@
+[database]
+db_username = postgres
+db_password = postgres
+db_host = localhost
+db_port = 5432
+db_name = pepys
+db_type = postgres
+[archive]
+user = pepys_admin
+password = 123456
+path =
+[local]
+parsers =
+basic_tests =
+enhanced_tests =

pepys_import/core/store/common_db.py

@@ -1,7 +1,11 @@
+from config import LOCAL_BASIC_TESTS, LOCAL_ENHANCED_TESTS
 from pepys_import.core.validators import constants as validation_constants
-
 from pepys_import.core.validators.basic_validator import BasicValidator
 from pepys_import.core.validators.enhanced_validator import EnhancedValidator
+from pepys_import.utils.import_utils import import_validators
+
+LOCAL_BASIC_VALIDATORS = import_validators(LOCAL_BASIC_TESTS)
+LOCAL_ENHANCED_VALIDATORS = import_validators(LOCAL_ENHANCED_TESTS)
 
 
 class SensorMixin:
@@ -156,13 +160,19 @@ def validate(
         elif validation_level == validation_constants.BASIC_LEVEL:
             for measurement in self.measurements[parser]:
                 BasicValidator(measurement, errors, parser)
+                for basic_validator in LOCAL_BASIC_VALIDATORS:
+                    basic_validator(measurement, errors, parser)
             if not errors:
                 return True
             return False
         elif validation_level == validation_constants.ENHANCED_LEVEL:
             for measurement in self.measurements[parser]:
                 BasicValidator(measurement, errors, parser)
+                for basic_validator in LOCAL_BASIC_VALIDATORS:
+                    basic_validator(measurement, errors, parser)
                 EnhancedValidator(measurement, errors, parser)
+                for enhanced_validator in LOCAL_ENHANCED_VALIDATORS:
+                    enhanced_validator(measurement, errors, parser)
             if not errors:
                 return True
             return False

pepys_import/file/file_processor.py

@@ -9,24 +9,25 @@
 from stat import S_IREAD
 
 from paths import IMPORTERS_DIRECTORY
+from config import ARCHIVE_PATH, LOCAL_PARSERS
 from pepys_import.core.store.data_store import DataStore
 from pepys_import.core.store.table_summary import TableSummary, TableSummarySet
 from pepys_import.file.highlighter.highlighter import HighlightedFile
 from pepys_import.file.importer import Importer
+from pepys_import.utils.import_utils import import_module_
 
 
 class FileProcessor:
     def __init__(self, filename=None):
         self.importers = []
         # Register local importers if any exists
-        local_importers_path = os.getenv("PEPYS_LOCAL_PARSERS")
-        if local_importers_path:
-            if not os.path.exists(local_importers_path):
+        if LOCAL_PARSERS:
+            if not os.path.exists(LOCAL_PARSERS):
                 print(
-                    f"No such file or directory: {local_importers_path}. Only core parsers are going to work."
+                    f"No such file or directory: {LOCAL_PARSERS}. Only core parsers are going to work."
                 )
             else:
-                self.load_importers_dynamically(local_importers_path)
+                self.load_importers_dynamically(LOCAL_PARSERS)
 
         if filename is None:
             self.filename = ":memory:"
@@ -36,11 +37,10 @@ def __init__(self, filename=None):
         self.input_files_path = None
         self.directory_path = None
         # Check if archive location environment variable exists
-        archive_path = os.getenv("PEPYS_ARCHIVE_LOCATION")
-        if archive_path:
-            if not os.path.exists(archive_path):
-                os.makedirs(archive_path)
-            self.output_path = archive_path
+        if ARCHIVE_PATH:
+            if not os.path.exists(ARCHIVE_PATH):
+                os.makedirs(ARCHIVE_PATH)
+            self.output_path = ARCHIVE_PATH
 
     def process(
         self, path: str, data_store: DataStore = None, descend_tree: bool = True
@@ -164,9 +164,8 @@ def process(
                 # loop through this path
                 for file in os.scandir(abs_path):
                     if file.is_file():
-                        current_path = os.path.join(abs_path, file)
                         processed_ctr = self.process_file(
-                            file, current_path, data_store, processed_ctr
+                            file, abs_path, data_store, processed_ctr
                         )
 
             states_sum = TableSummary(data_store.session, data_store.db_classes.State)
@@ -190,7 +189,7 @@ def process_file(self, file, current_path, data_store, processed_ctr):
         # make copy of list of importers
         good_importers = self.importers.copy()
 
-        full_path = os.path.join(current_path, file)
+        full_path = os.path.join(current_path, basename)
         # print("Checking:" + str(full_path))
 
         # start with file suffixes
@@ -274,7 +273,7 @@ def process_file(self, file, current_path, data_store, processed_ctr):
                 ) as f:
                     f.write("\n".join(log))
                 # move original file to output folder
-                new_path = os.path.join(self.input_files_path, file)
+                new_path = os.path.join(self.input_files_path, basename)
                 shutil.move(full_path, new_path)
                 # make it read-only
                 os.chmod(new_path, S_IREAD)
@@ -309,14 +308,7 @@ def load_importers_dynamically(self, path=IMPORTERS_DIRECTORY):
             for file in os.scandir(path):
                 # import file using its name and full path
                 if file.is_file():
-                    spec = importlib.util.spec_from_file_location(file.name, file.path)
-                    module = importlib.util.module_from_spec(spec)
-                    sys.modules[file.name] = module
-                    spec.loader.exec_module(module)
-                    # extract classes with this format: (class name, class)
-                    classes = inspect.getmembers(
-                        sys.modules[module.__name__], inspect.isclass
-                    )
+                    classes = import_module_(file)
                     for name, class_ in classes:
                         # continue only if it's a concrete class that inherits Importer
                         if issubclass(class_, Importer) and not inspect.isabstract(

pepys_import/utils/config_utils.py

@@ -0,0 +1,15 @@
+from string import ascii_lowercase, ascii_uppercase
+
+
+def process(text):
+    al = ascii_lowercase
+    au = ascii_uppercase
+    new = ""
+    for i in text:
+        if str.islower(i):
+            new += al[(al.find(i) + 13) % len(al)]
+        elif str.isupper(i):
+            new += au[(au.find(i) + 13) % len(au)]
+        else:
+            new += i
+    return new

pepys_import/utils/import_utils.py

@@ -0,0 +1,41 @@
+import importlib.util
+import inspect
+import os
+import sys
+
+
+def import_module_(file):
+    """Import modules using file object's full path and name.
+
+    :param file: File object, which has name and full path attributes
+    :type file: File
+    """
+    spec = importlib.util.spec_from_file_location(file.name, file.path)
+    # If spec is none, it means that it is not a module, return an empty list
+    if spec is None:
+        return list()
+    module = importlib.util.module_from_spec(spec)
+    sys.modules[file.name] = module
+    spec.loader.exec_module(module)
+    # extract classes with this format: (class name, class)
+    classes = inspect.getmembers(sys.modules[module.__name__], inspect.isclass)
+    return classes
+
+
+def import_validators(path):
+    """Import validators in the given path.
+
+    :param path: Path to the directory that has validators
+    :type path: String
+    :return:
+    """
+    validators = list()
+    if os.path.exists(path):
+        for file in os.scandir(path):
+            # import file using its name and full path
+            if file.is_file():
+                classes = import_module_(file)
+                for name, class_ in classes:
+                    print(inspect.signature(class_))
+                    validators.append(class_)
+    return validators

tests/config_file_tests/basic_tests/basic.py

@@ -0,0 +1,7 @@
+class BasicTestValidator:
+    def __init__(self, measurement_object, errors, parser_name):
+        self.error_type = parser_name + f" - Test Basic Validation Error"
+        self.errors = errors
+
+    def validate(self):
+        return True

tests/config_file_tests/enhanced_tests/enhanced.py

@@ -0,0 +1,7 @@
+class EnhancedTestValidator:
+    def __init__(self, measurement_object, errors, parser_name):
+        self.error_type = parser_name + f" - Test Enhanced Validation Error"
+        self.errors = errors
+
+    def validate(self):
+        return True

tests/config_file_tests/example_config/config.ini

@@ -0,0 +1,15 @@
+[database]
+db_username = _Test User_
+db_password = _123456_
+db_host = localhost
+db_port = 5432
+db_name = test
+db_type = postgres
+[archive]
+user = _user_
+password = _password_
+path = path/to/archive
+[local]
+parsers = path/to/parser
+basic_tests = path/to/basic/tests
+enhanced_tests = path/to/enhanced/tests

tests/config_file_tests/test_config_file.py

@@ -0,0 +1,97 @@
+import config
+import unittest
+import os
+import pytest
+
+from contextlib import redirect_stdout
+from io import StringIO
+from unittest.mock import patch
+from importlib import reload
+
+from pepys_import.file.file_processor import FileProcessor
+from pepys_import.core.store import common_db
+
+DIRECTORY_PATH = os.path.dirname(__file__)
+TEST_IMPORTER_PATH = os.path.join(DIRECTORY_PATH, "parsers")
+BAD_IMPORTER_PATH = os.path.join(DIRECTORY_PATH, "bad_path")
+OUTPUT_PATH = os.path.join(DIRECTORY_PATH, "output")
+CONFIG_FILE_PATH = os.path.join(DIRECTORY_PATH, "example_config", "config.ini")
+BASIC_PARSERS_PATH = os.path.join(DIRECTORY_PATH, "basic_tests")
+ENHANCED_PARSERS_PATH = os.path.join(DIRECTORY_PATH, "enhanced_tests")
+
+
+class ConfigVariablesTestCase(unittest.TestCase):
+    @patch.dict(os.environ, {"PEPYS_CONFIG_FILE": CONFIG_FILE_PATH})
+    def test_config_variables(self):
+        reload(config)
+        assert config.DB_USERNAME == "Grfg Hfre"
+        assert config.DB_PASSWORD == "123456"
+        assert config.DB_HOST == "localhost"
+        assert config.DB_PORT == 5432
+        assert config.DB_NAME == "test"
+        assert config.DB_TYPE == "postgres"
+        assert config.ARCHIVE_USER == "hfre"
+        assert config.ARCHIVE_PASSWORD == "cnffjbeq"
+        assert config.ARCHIVE_PATH == "path/to/archive"
+        assert config.LOCAL_PARSERS == "path/to/parser"
+        assert config.LOCAL_BASIC_TESTS == "path/to/basic/tests"
+        assert config.LOCAL_ENHANCED_TESTS == "path/to/enhanced/tests"
+
+    @patch.dict(os.environ, {"PEPYS_CONFIG_FILE": BAD_IMPORTER_PATH})
+    def test_wrong_file_path(self):
+        # No such file exception
+        with pytest.raises(Exception):
+            reload(config)
+
+    @patch.dict(os.environ, {"PEPYS_CONFIG_FILE": TEST_IMPORTER_PATH})
+    def test_wrong_file_path_2(self):
+        # Your environment variable doesn't point to a file exception
+        with pytest.raises(Exception):
+            reload(config)
+
+
+class FileProcessorVariablesTestCase(unittest.TestCase):
+    @patch("pepys_import.file.file_processor.LOCAL_PARSERS", TEST_IMPORTER_PATH)
+    def test_pepys_local_parsers(self):
+        file_processor = FileProcessor()
+
+        assert len(file_processor.importers) == 1
+        assert file_processor.importers[0].name == "Test Importer"
+
+    @patch("pepys_import.file.file_processor.LOCAL_PARSERS", BAD_IMPORTER_PATH)
+    def test_bad_pepys_local_parsers_path(self):
+        temp_output = StringIO()
+        with redirect_stdout(temp_output):
+            file_processor = FileProcessor()
+        output = temp_output.getvalue()
+
+        assert len(file_processor.importers) == 0
+        assert (
+            output
+            == f"No such file or directory: {BAD_IMPORTER_PATH}. Only core parsers are going to work.\n"
+        )
+
+    @patch("pepys_import.file.file_processor.ARCHIVE_PATH", OUTPUT_PATH)
+    def test_pepys_archive_location(self):
+        file_processor = FileProcessor()
+        assert os.path.exists(OUTPUT_PATH) is True
+        assert file_processor.output_path == OUTPUT_PATH
+        # Remove the test_output directory
+        os.rmdir(OUTPUT_PATH)
+
+
+class CommonDBVariablesTestCase(unittest.TestCase):
+    @patch("config.LOCAL_BASIC_TESTS", BASIC_PARSERS_PATH)
+    @patch("config.LOCAL_ENHANCED_TESTS", ENHANCED_PARSERS_PATH)
+    def test_local_parser_tests(self):
+        assert not common_db.LOCAL_BASIC_VALIDATORS
+        assert not common_db.LOCAL_ENHANCED_VALIDATORS
+
+        # reload common_db module
+        reload(common_db)
+        assert len(common_db.LOCAL_BASIC_VALIDATORS) == 1
+        assert len(common_db.LOCAL_ENHANCED_VALIDATORS) == 1
+
+
+if __name__ == "__main__":
+    unittest.main()