feat: allow remote database configuration (anitab-org#207)

* chore: allow amazon rds database usage
* feat: 3 environments access mysqldb
* updated requirements.txt and addedurl for staging db
* updated requirements.txt
* feat: update config to support any type of remote database
* fix: build_uri functioon and test
* fix: remove dependency causing build fail
* Update deploy.sh to add more env variables

README.md

@@ -33,7 +33,7 @@ or
 make sure you exported the following [environment variables](https://github.com/systers/mentorship-backend/wiki/Environment-Variables):
 
 ```
-export FLASK_ENVIRONMENT_CONFIG=<dev-or-test-or-prod>
+export FLASK_ENVIRONMENT_CONFIG=<local-or-dev-or-test-or-prod-or-stag>
 export SECRET_KEY=<your-secret-key>
 export SECURITY_PASSWORD_SALT=<your-security-password-salt>
 export MAIL_DEFAULT_SENDER=<mail-default-sender>
@@ -42,6 +42,15 @@ export APP_MAIL_USERNAME=<app-mail-username>
 export APP_MAIL_PASSWORD=<app-mail-password>
 ```
 
+If you're testing any environment other than "local", then you have to also set these other variables:
+```
+export DB_TYPE=<database_type>
+export DB_USERNAME=<database_username>
+export DB_PASSWORD=<database_password>
+export DB_ENDPOINT=<database_endpoint>
+export DB_NAME=<database_name>
+```
+
 5. Run the app:
 `python run.py`
 

config.py

@@ -3,12 +3,22 @@
 
 
 class BaseConfig(object):
+    """Base configuration."""
     DEBUG = False
     TESTING = False
 
     # SQLAlchemy settings
-    SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
     SQLALCHEMY_TRACK_MODIFICATIONS = False
+    SQLALCHEMY_POOL_RECYCLE = 3600
+
+    # Example:
+    # MySQL: mysql+pymysql://{db_user}:{db_password}@{db_endpoint}/{db_name}
+    # SQLite: sqlite:///local_data.db
+    DB_TYPE = os.getenv('DB_TYPE')
+    DB_USERNAME = os.getenv('DB_USERNAME')
+    DB_PASSWORD = os.getenv('DB_PASSWORD')
+    DB_ENDPOINT = os.getenv('DB_ENDPOINT')
+    DB_NAME = os.getenv('DB_NAME')
 
     UNVERIFIED_USER_THRESHOLD = 2592000  # 30 days
 
@@ -25,7 +35,6 @@ class BaseConfig(object):
 
     BCRYPT_LOG_ROUNDS = 13
     WTF_CSRF_ENABLED = True
-    # TODO put this in dev only?
 
     DEBUG_TB_ENABLED = False
     DEBUG_TB_INTERCEPT_REDIRECTS = False
@@ -43,44 +52,68 @@ class BaseConfig(object):
     # mail accounts
     MAIL_DEFAULT_SENDER = os.getenv('MAIL_DEFAULT_SENDER')
 
+    @staticmethod
+    def build_db_uri(
+        db_type_arg=DB_TYPE,
+        db_user_arg=DB_USERNAME,
+        db_password_arg=DB_PASSWORD,
+        db_endpoint_arg=DB_ENDPOINT,
+        db_name_arg=DB_NAME
+        ):
+        """Build remote database uri using specific environment variables."""
+
+        return '{db_type}://{db_user}:{db_password}@{db_endpoint}/{db_name}'\
+            .format(db_type=db_type_arg,
+                    db_user=db_user_arg,
+                    db_password=db_password_arg,
+                    db_endpoint=db_endpoint_arg,
+                    db_name=db_name_arg)
 
-class ProductionConfig(BaseConfig):
-    ENV = 'production'
 
-    # SQLALCHEMY_DATABASE_URI = 'mysql://user@localhost/foo'
-    SQLALCHEMY_DATABASE_URI = 'sqlite:///prod_data.db'
+class ProductionConfig(BaseConfig):
+    """Production configuration."""
+    SQLALCHEMY_DATABASE_URI = BaseConfig.build_db_uri()
 
 
 class DevelopmentConfig(BaseConfig):
-    ENV = 'development'
+    """Development configuration."""
     DEBUG = True
+    SQLALCHEMY_DATABASE_URI = BaseConfig.build_db_uri()
 
-    SQLALCHEMY_DATABASE_URI = 'sqlite:///dev_data.db'
 
-    # mail accounts
-    MAIL_DEFAULT_SENDER = 'certain@example.com'
+class StagingConfig(BaseConfig):
+    """Staging configuration."""
+    DEBUG = True
+    SQLALCHEMY_DATABASE_URI = BaseConfig.build_db_uri()
 
 
-class TestingConfig(BaseConfig):
-    ENV = 'testing'
+class LocalConfig(BaseConfig):
+    """Local configuration."""
     DEBUG = True
+
+    # Using a local sqlite database 
+    SQLALCHEMY_DATABASE_URI = 'sqlite:///local_data.db'
+
+
+class TestingConfig(BaseConfig):
+    """Testing configuration."""
     TESTING = True
 
     # Use in-memory SQLite database for testing
     SQLALCHEMY_DATABASE_URI = 'sqlite://'
-    # SQLALCHEMY_DATABASE_URI = 'sqlite:///:memory:'
-    # SQLALCHEMY_DATABASE_URI = 'sqlite:///test_data.db'
 
 
 def get_env_config():
     flask_config_name = os.getenv('FLASK_ENVIRONMENT_CONFIG', 'dev')
-    if flask_config_name not in ['prod', 'test', 'dev']:
+    if flask_config_name not in ['prod', 'test', 'dev', 'local', 'stag']:
         raise ValueError('The environment config value has to be within these values: prod, dev, test.')
     return CONFIGURATION_MAPPER[flask_config_name]
 
 
 CONFIGURATION_MAPPER = {
     'dev': 'config.DevelopmentConfig',
-    'test': 'config.TestingConfig',
-    'prod': 'config.ProductionConfig'
+    'prod': 'config.ProductionConfig',
+    'stag': 'config.StagingConfig',
+    'local': 'config.LocalConfig',
+    'test': 'config.TestingConfig'
 }

deploy.sh

@@ -36,7 +36,7 @@ echo "aws_access_key_id = $AWS_ACCESS_ID" >> ~/.aws/config
 echo "aws_secret_access_key = $AWS_SECRET_KEY" >> ~/.aws/config
 
 # Add environment variables
-eb setenv FLASK_ENVIRONMENT_CONFIG=$FLASK_ENVIRONMENT_CONFIG MAIL_DEFAULT_SENDER=$MAIL_DEFAULT_SENDER MAIL_SERVER=$MAIL_SERVER APP_MAIL_USERNAME=$APP_MAIL_USERNAME APP_MAIL_PASSWORD=$APP_MAIL_PASSWORD SECRET_KEY=$SECRET_KEY SECURITY_PASSWORD_SALT=$SECURITY_PASSWORD_SALT
+eb setenv FLASK_ENVIRONMENT_CONFIG=$FLASK_ENVIRONMENT_CONFIG MAIL_DEFAULT_SENDER=$MAIL_DEFAULT_SENDER MAIL_SERVER=$MAIL_SERVER APP_MAIL_USERNAME=$APP_MAIL_USERNAME APP_MAIL_PASSWORD=$APP_MAIL_PASSWORD SECRET_KEY=$SECRET_KEY SECURITY_PASSWORD_SALT=$SECURITY_PASSWORD_SALT DB_TYPE=$DB_TYPE DB_USERNAME=$DB_USERNAME DB_PASSWORD=$DB_PASSWORD DB_ENDPOINT=$DB_ENDPOINT DB_NAME=$DB_NAME 
 
 # Publishing
 echo "Publishing to '$SERVER' server"

requirements.txt

@@ -1,16 +1,19 @@
 aniso8601==3.0.0
 APScheduler==3.5.1
+asn1crypto==0.24.0
 awsebcli==3.14.1
 blessed==1.15.0
 blinker==1.4
 botocore==1.10.48
 cached-property==1.4.3
 cement==2.8.2
 certifi==2018.4.16
+cffi==1.11.5
 chardet==3.0.4
 click==6.7
 colorama==0.3.9
 coverage==4.5.1
+cryptography==2.3
 docker==3.4.0
 docker-compose==1.21.2
 docker-pycreds==0.3.0
@@ -30,7 +33,10 @@ jmespath==0.9.3
 jsonschema==2.6.0
 MarkupSafe==1.0
 pathspec==0.5.5
+psycopg2-binary==2.8.3
+pycparser==2.18
 PyJWT==1.4.2
+PyMySQL==0.9.2
 python-dateutil==2.7.3
 python-dotenv==0.8.2
 pytz==2018.4

tests/test_app_config.py

@@ -4,6 +4,8 @@
 
 from flask import current_app
 from flask_testing import TestCase
+
+from config import BaseConfig
 from run import application
 
 
@@ -17,7 +19,7 @@ def create_app(self):
 
     def test_app_testing_config(self):
         self.assertIsNotNone(application.config['SECRET_KEY'])
-        self.assertTrue(application.config['DEBUG'])
+        self.assertFalse(application.config['DEBUG'])
         self.assertTrue(application.config['TESTING'])
         self.assertFalse(application.config['SQLALCHEMY_TRACK_MODIFICATIONS'])
         self.assertEqual('sqlite://', application.config['SQLALCHEMY_DATABASE_URI'])
@@ -27,21 +29,73 @@ def test_app_testing_config(self):
         self.assertEqual(timedelta(weeks=1), application.config['JWT_ACCESS_TOKEN_EXPIRES'])
         self.assertEqual(timedelta(weeks=4), application.config['JWT_REFRESH_TOKEN_EXPIRES'])
 
+    def test_get_bd_uri_function(self):
+
+        expected_result = 'db_type_example://db_user_example:db_password_example@db_endpoint_example/db_name_example'
+        actual_result = BaseConfig.build_db_uri(db_type_arg='db_type_example',
+                                              db_user_arg='db_user_example',
+                                              db_password_arg='db_password_example',
+                                              db_endpoint_arg='db_endpoint_example',
+                                              db_name_arg='db_name_example')
+        self.assertEqual(expected_result, actual_result)
+
 
 class TestDevelopmentConfig(TestCase):
     def create_app(self):
         application.config.from_object('config.DevelopmentConfig')
 
         secret_key = os.getenv('SECRET_KEY', None)
         application.config['SECRET_KEY'] = secret_key if secret_key else 'TEST_SECRET_KEY'
+
+        return application
+
+    def test_app_development_config(self):
+        self.assertIsNotNone(application.config['SECRET_KEY'])
+        self.assertTrue(application.config['DEBUG'])
+        self.assertFalse(application.config['TESTING'])
+        self.assertFalse(application.config['SQLALCHEMY_TRACK_MODIFICATIONS'])
+        # self.assertEqual('mysql_something', application.config['SQLALCHEMY_DATABASE_URI'])
+        self.assertIsNotNone(current_app)
+
+        # testing JWT configurations
+        self.assertEqual(timedelta(weeks=1), application.config['JWT_ACCESS_TOKEN_EXPIRES'])
+
+
+class TestStagingConfig(TestCase):
+    def create_app(self):
+        application.config.from_object('config.StagingConfig')
+
+        secret_key = os.getenv('SECRET_KEY', None)
+        application.config['SECRET_KEY'] = secret_key if secret_key else 'TEST_SECRET_KEY'
+
         return application
 
     def test_app_development_config(self):
         self.assertIsNotNone(application.config['SECRET_KEY'])
         self.assertTrue(application.config['DEBUG'])
         self.assertFalse(application.config['TESTING'])
         self.assertFalse(application.config['SQLALCHEMY_TRACK_MODIFICATIONS'])
-        self.assertEqual('sqlite:///dev_data.db', application.config['SQLALCHEMY_DATABASE_URI'])
+        # self.assertEqual('mysql_something', application.config['SQLALCHEMY_DATABASE_URI'])
+        self.assertIsNotNone(current_app)
+
+        # testing JWT configurations
+        self.assertEqual(timedelta(weeks=1), application.config['JWT_ACCESS_TOKEN_EXPIRES'])
+
+
+class TestLocalConfig(TestCase):
+    def create_app(self):
+        application.config.from_object('config.LocalConfig')
+
+        secret_key = os.getenv('SECRET_KEY', None)
+        application.config['SECRET_KEY'] = secret_key if secret_key else 'TEST_SECRET_KEY'
+        return application
+
+    def test_app_development_config(self):
+        self.assertIsNotNone(application.config['SECRET_KEY'])
+        self.assertTrue(application.config['DEBUG'])
+        self.assertFalse(application.config['TESTING'])
+        self.assertFalse(application.config['SQLALCHEMY_TRACK_MODIFICATIONS'])
+        self.assertEqual('sqlite:///local_data.db', application.config['SQLALCHEMY_DATABASE_URI'])
         self.assertIsNotNone(current_app)
 
         # testing JWT configurations
@@ -55,14 +109,15 @@ def create_app(self):
 
         secret_key = os.getenv('SECRET_KEY', None)
         application.config['SECRET_KEY'] = secret_key if secret_key else 'TEST_SECRET_KEY'
+
         return application
 
     def test_app_production_config(self):
         self.assertIsNotNone(application.config['SECRET_KEY'])
         self.assertFalse(application.config['DEBUG'])
         self.assertFalse(application.config['TESTING'])
         self.assertFalse(application.config['SQLALCHEMY_TRACK_MODIFICATIONS'])
-        self.assertEqual('sqlite:///prod_data.db', application.config['SQLALCHEMY_DATABASE_URI'])
+        # self.assertEqual('mysql_something', application.config['SQLALCHEMY_DATABASE_URI'])
         self.assertIsNotNone(current_app)
 
         # testing JWT configurations