From 92848dc33ecfcc010e71ba5af97c209bb95bee06 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 13 Mar 2024 00:43:15 -0400 Subject: [PATCH 01/14] Preparing release mgmt changes --- TEST.md | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 TEST.md diff --git a/TEST.md b/TEST.md new file mode 100644 index 00000000..2eb740c5 --- /dev/null +++ b/TEST.md @@ -0,0 +1,8 @@ +# Things to get Done to improve Release management + +- Add Code Coverage using pytest-cov library +- Add Rule that PR must include Tests +- Add PR issue Template +- Add pytest for 3.10 and 3.11 +- generate pytest-cov report +- add ruff and black to pre-commit file \ No newline at end of file From a3fa486be652de29115ee81b9047d83f4b42c42f Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 25 Mar 2024 10:27:16 -0400 Subject: [PATCH 02/14] Adding Pyupgrade to reformat code --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7deaef31..5c27d6fe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,3 +10,7 @@ repos: rev: v2.4.0 hooks: - id: setup-cfg-fmt +- repo: https://github.com/asottile/pyupgrade + rev: v3.15.2 + hooks: + - id: pyupgrade From e97914bb1f1ba54aee26a533e14c9bb9424e26c4 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 25 Mar 2024 11:09:23 -0400 Subject: [PATCH 03/14] updated the code based on pyupgrade and ruff linting --- .pre-commit-config.yaml | 19 +++++++++++++++++++ schemachange/cli.py | 23 +++++++++++++++-------- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5c27d6fe..9639d152 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,3 +14,22 @@ repos: rev: v3.15.2 hooks: - id: pyupgrade +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.3.4 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format +# Using this mirror lets us use mypyc-compiled black, which is about 2x faster +# - repo: https://github.com/psf/black-pre-commit-mirror +# rev: 24.3.0 +# hooks: +# - id: black +# # It is recommended to specify the latest version of Python +# # supported by your project here, or alternatively use +# # pre-commit's default_language_version, see +# # https://pre-commit.com/#top_level-default_language_version +# language_version: python3.10 \ No newline at end of file diff --git a/schemachange/cli.py b/schemachange/cli.py index 31745f43..5bd8c99d 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -253,7 +253,8 @@ def get_oauth_token(self): token_name = self.oauth_config['token-response-name'] response = requests.post(**req_info) resJsonDict =json.loads(response.text) - try: return resJsonDict[token_name] + try: + return resJsonDict[token_name] except KeyError: errormessage = _err_oauth_tk_nm.format( keys = ', '.join(resJsonDict.keys()), @@ -479,15 +480,15 @@ def apply_change_script(self, script, script_content, change_history_table): def deploy_command(config): # Make sure we have the required connection info, all of the below needs to be present. - req_args = set(['snowflake_account','snowflake_user','snowflake_role','snowflake_warehouse']) + req_args = {'snowflake_account','snowflake_user','snowflake_role','snowflake_warehouse'} provided_args = {k:v for (k,v) in config.items() if v} missing_args = req_args -provided_args.keys() if len(missing_args)>0: raise ValueError(_err_args_missing % ', '.join({s.replace('_', ' ') for s in missing_args})) #ensure an authentication method is specified / present. one of the below needs to be present. - req_env_var = set(['SNOWFLAKE_PASSWORD', 'SNOWSQL_PWD','SNOWFLAKE_PRIVATE_KEY_PATH','SNOWFLAKE_AUTHENTICATOR']) - if len((req_env_var - dict(os.environ).keys()))==len(req_env_var): + req_env_var = {'SNOWFLAKE_PASSWORD', 'SNOWSQL_PWD','SNOWFLAKE_PRIVATE_KEY_PATH','SNOWFLAKE_AUTHENTICATOR'} + if len(req_env_var - dict(os.environ).keys())==len(req_env_var): raise ValueError(_err_env_missing) # Log some additional details @@ -600,14 +601,20 @@ def render_command(config, script_path): print("Checksum %s" % checksum) print(content) +def alphanum_convert(text:str): + result = None + if text.isdigit(): + result = int(text) + else: + result = text.lower() + return result # This function will return a list containing the parts of the key (split by number parts) # Each number is converted to and integer and string parts are left as strings # This will enable correct sorting in python when the lists are compared # e.g. get_alphanum_key('1.2.2') results in ['', 1, '.', 2, '.', 2, ''] def get_alphanum_key(key): - convert = lambda text: int(text) if text.isdigit() else text.lower() - alphanum_key = [ convert(c) for c in re.split('([0-9]+)', key) ] + alphanum_key = [ alphanum_convert(c) for c in re.split('([0-9]+)', key) ] return alphanum_key def sorted_alphanumeric(data): @@ -664,7 +671,7 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf "vars":{}, "create_change_history_table":False, "autocommit":False, "verbose":False, \ "dry_run":False , "query_tag":None , "oauth_config":None } #insert defualt values for items not populated - config.update({ k:v for (k,v) in config_defaults.items() if not k in config.keys()}) + config.update({ k:v for (k,v) in config_defaults.items() if k not in config.keys()}) # Validate folder paths if 'root_folder' in config: @@ -678,7 +685,7 @@ def get_schemachange_config(config_file_path, root_folder, modules_folder, snowf raise ValueError(_err_invalid_folder.format(folder_type='modules', path=config['modules_folder'])) if config['vars']: # if vars is configured wrong in the config file it will come through as a string - if type(config['vars']) is not dict: + if not isinstance(config['vars'], dict): raise ValueError(_err_vars_config) # the variable schema change has been reserved From c2c97fbfaf46cf6531c988560ac786b467328cbd Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 25 Mar 2024 11:11:49 -0400 Subject: [PATCH 04/14] linting done using black formatter --- schemachange/cli.py | 1997 +++++++++++++++++++++++++------------------ 1 file changed, 1179 insertions(+), 818 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index 5bd8c99d..f321b789 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -19,888 +19,1249 @@ from jinja2.loaders import BaseLoader from pandas import DataFrame -#region Global Variables +# region Global Variables # metadata -_schemachange_version = '3.6.1' -_config_file_name = 'schemachange-config.yml' -_metadata_database_name = 'METADATA' -_metadata_schema_name = 'SCHEMACHANGE' -_metadata_table_name = 'CHANGE_HISTORY' -_snowflake_application_name = 'schemachange' +_schemachange_version = "3.6.1" +_config_file_name = "schemachange-config.yml" +_metadata_database_name = "METADATA" +_metadata_schema_name = "SCHEMACHANGE" +_metadata_table_name = "CHANGE_HISTORY" +_snowflake_application_name = "schemachange" # messages -_err_jinja_env_var = "Could not find environmental variable %s and no default" \ - + " value was provided" -_err_oauth_tk_nm = 'Response Json contains keys: {keys} \n but not {key}' -_err_oauth_tk_err = '\n error description: {desc}' -_err_no_auth_mthd = "Unable to find connection credentials for Okta, private key, " \ - + "password, Oauth or Browser authentication" -_err_unsupported_auth_mthd = "'{unsupported_authenticator}' is not supported authenticator option. " \ - + "Choose from externalbrowser, oauth, https://.okta.com. Using default value = 'snowflake'" -_warn_password = "The SNOWSQL_PWD environment variable is deprecated and will" \ - + " be removed in a later version of schemachange. Please use SNOWFLAKE_PASSWORD instead." -_warn_password_dup = "Environment variables SNOWFLAKE_PASSWORD and SNOWSQL_PWD are " \ - + " both present, using SNOWFLAKE_PASSWORD" -_err_args_missing ="Missing config values. The following config values are required: %s " -_err_env_missing ="Missing environment variable(s). \nSNOWFLAKE_PASSWORD must be defined for " \ - + "password authentication. \nSNOWFLAKE_PRIVATE_KEY_PATH and (optional) " \ - + "SNOWFLAKE_PRIVATE_KEY_PASSPHRASE must be defined for private key authentication. " \ - + "\nSNOWFLAKE_AUTHENTICATOR must be defined is using Oauth, OKTA or external Browser Authentication." -_log_config_details = "Using Snowflake account {snowflake_account}\nUsing default role " \ - + "{snowflake_role}\nUsing default warehouse {snowflake_warehouse}\nUsing default " \ - + "database {snowflake_database}" \ - + "schema {snowflake_schema}" -_log_ch_use = "Using change history table {database_name}.{schema_name}.{table_name} " \ - + "(last altered {last_altered})" -_log_ch_create = "Created change history table {database_name}.{schema_name}.{table_name}" -_err_ch_missing = "Unable to find change history table {database_name}.{schema_name}.{table_name}" -_log_ch_max_version = "Max applied change script version: {max_published_version_display}" -_log_skip_v = "Skipping change script {script_name} because it's older than the most recently " \ - + "applied change ({max_published_version})" -_log_skip_r ="Skipping change script {script_name} because there is no change since the last " \ - + "execution" -_log_apply = "Applying change script {script_name}" -_log_apply_set_complete = "Successfully applied {scripts_applied} change scripts (skipping " \ - + "{scripts_skipped}) \nCompleted successfully" +_err_jinja_env_var = ( + "Could not find environmental variable %s and no default" + " value was provided" +) +_err_oauth_tk_nm = "Response Json contains keys: {keys} \n but not {key}" +_err_oauth_tk_err = "\n error description: {desc}" +_err_no_auth_mthd = ( + "Unable to find connection credentials for Okta, private key, " + + "password, Oauth or Browser authentication" +) +_err_unsupported_auth_mthd = ( + "'{unsupported_authenticator}' is not supported authenticator option. " + + "Choose from externalbrowser, oauth, https://.okta.com. Using default value = 'snowflake'" +) +_warn_password = ( + "The SNOWSQL_PWD environment variable is deprecated and will" + + " be removed in a later version of schemachange. Please use SNOWFLAKE_PASSWORD instead." +) +_warn_password_dup = ( + "Environment variables SNOWFLAKE_PASSWORD and SNOWSQL_PWD are " + + " both present, using SNOWFLAKE_PASSWORD" +) +_err_args_missing = ( + "Missing config values. The following config values are required: %s " +) +_err_env_missing = ( + "Missing environment variable(s). \nSNOWFLAKE_PASSWORD must be defined for " + + "password authentication. \nSNOWFLAKE_PRIVATE_KEY_PATH and (optional) " + + "SNOWFLAKE_PRIVATE_KEY_PASSPHRASE must be defined for private key authentication. " + + "\nSNOWFLAKE_AUTHENTICATOR must be defined is using Oauth, OKTA or external Browser Authentication." +) +_log_config_details = ( + "Using Snowflake account {snowflake_account}\nUsing default role " + + "{snowflake_role}\nUsing default warehouse {snowflake_warehouse}\nUsing default " + + "database {snowflake_database}" + + "schema {snowflake_schema}" +) +_log_ch_use = ( + "Using change history table {database_name}.{schema_name}.{table_name} " + + "(last altered {last_altered})" +) +_log_ch_create = ( + "Created change history table {database_name}.{schema_name}.{table_name}" +) +_err_ch_missing = ( + "Unable to find change history table {database_name}.{schema_name}.{table_name}" +) +_log_ch_max_version = ( + "Max applied change script version: {max_published_version_display}" +) +_log_skip_v = ( + "Skipping change script {script_name} because it's older than the most recently " + + "applied change ({max_published_version})" +) +_log_skip_r = ( + "Skipping change script {script_name} because there is no change since the last " + + "execution" +) +_log_apply = "Applying change script {script_name}" +_log_apply_set_complete = ( + "Successfully applied {scripts_applied} change scripts (skipping " + + "{scripts_skipped}) \nCompleted successfully" +) _err_vars_config = "vars did not parse correctly, please check its configuration" -_err_vars_reserved = "The variable schemachange has been reserved for use by schemachange, " \ - + "please use a different name" -_err_invalid_folder = "Invalid {folder_type} folder: {path}" -_err_dup_scripts = "The script name {script_name} exists more than once (first_instance " \ - + "{first_path}, second instance {script_full_path})" -_err_dup_scripts_version = "The script version {script_version} exists more than once " \ - + "(second instance {script_full_path})" -_err_invalid_cht = 'Invalid change history table name: %s' -_log_auth_type ="Proceeding with %s authentication" -_log_pk_enc ="No private key passphrase provided. Assuming the key is not encrypted." -_log_okta_ep ="Okta Endpoint: %s" - -#endregion Global Variables +_err_vars_reserved = ( + "The variable schemachange has been reserved for use by schemachange, " + + "please use a different name" +) +_err_invalid_folder = "Invalid {folder_type} folder: {path}" +_err_dup_scripts = ( + "The script name {script_name} exists more than once (first_instance " + + "{first_path}, second instance {script_full_path})" +) +_err_dup_scripts_version = ( + "The script version {script_version} exists more than once " + + "(second instance {script_full_path})" +) +_err_invalid_cht = "Invalid change history table name: %s" +_log_auth_type = "Proceeding with %s authentication" +_log_pk_enc = "No private key passphrase provided. Assuming the key is not encrypted." +_log_okta_ep = "Okta Endpoint: %s" + +# endregion Global Variables -class JinjaEnvVar(jinja2.ext.Extension): - """ - Extends Jinja Templates with access to environmental variables - """ - def __init__(self, environment: jinja2.Environment): - super().__init__(environment) - - # add globals - environment.globals["env_var"] = JinjaEnvVar.env_var - @staticmethod - def env_var(env_var: str, default: Optional[str] = None) -> str: +class JinjaEnvVar(jinja2.ext.Extension): """ - Returns the value of the environmental variable or the default. + Extends Jinja Templates with access to environmental variables """ - result = default - if env_var in os.environ: - result = os.environ[env_var] - if result is None: - raise ValueError(_err_jinja_env_var % env_var) + def __init__(self, environment: jinja2.Environment): + super().__init__(environment) - return result + # add globals + environment.globals["env_var"] = JinjaEnvVar.env_var + @staticmethod + def env_var(env_var: str, default: Optional[str] = None) -> str: + """ + Returns the value of the environmental variable or the default. + """ + result = default + if env_var in os.environ: + result = os.environ[env_var] -class JinjaTemplateProcessor: - _env_args = {"undefined":jinja2.StrictUndefined,"autoescape":False, "extensions":[JinjaEnvVar]} - - def __init__(self, project_root: str, modules_folder: str = None): - loader: BaseLoader - if modules_folder: - loader = jinja2.ChoiceLoader( - [ - jinja2.FileSystemLoader(project_root), - jinja2.PrefixLoader({"modules": jinja2.FileSystemLoader(modules_folder)}), - ] - ) - else: - loader = jinja2.FileSystemLoader(project_root) - self.__environment = jinja2.Environment(loader=loader, **self._env_args) - self.__project_root = project_root + if result is None: + raise ValueError(_err_jinja_env_var % env_var) - def list(self): - return self.__environment.list_templates() + return result - def override_loader(self, loader: jinja2.BaseLoader): - # to make unit testing easier - self.__environment = jinja2.Environment(loader=loader, **self._env_args) - def render(self, script: str, vars: Dict[str, Any], verbose: bool) -> str: - if not vars: - vars = {} - # jinja needs posix path - posix_path = pathlib.Path(script).as_posix() - template = self.__environment.get_template(posix_path) - content = template.render(**vars).strip() - content = content[:-1] if content.endswith(';') else content - return content +class JinjaTemplateProcessor: + _env_args = { + "undefined": jinja2.StrictUndefined, + "autoescape": False, + "extensions": [JinjaEnvVar], + } - def relpath(self, file_path: str): - return os.path.relpath(file_path, self.__project_root) + def __init__(self, project_root: str, modules_folder: str = None): + loader: BaseLoader + if modules_folder: + loader = jinja2.ChoiceLoader( + [ + jinja2.FileSystemLoader(project_root), + jinja2.PrefixLoader( + {"modules": jinja2.FileSystemLoader(modules_folder)} + ), + ] + ) + else: + loader = jinja2.FileSystemLoader(project_root) + self.__environment = jinja2.Environment(loader=loader, **self._env_args) + self.__project_root = project_root + def list(self): + return self.__environment.list_templates() -class SecretManager: - """ - Provides the ability to redact secrets - """ - __singleton: 'SecretManager' + def override_loader(self, loader: jinja2.BaseLoader): + # to make unit testing easier + self.__environment = jinja2.Environment(loader=loader, **self._env_args) + + def render(self, script: str, vars: Dict[str, Any], verbose: bool) -> str: + if not vars: + vars = {} + # jinja needs posix path + posix_path = pathlib.Path(script).as_posix() + template = self.__environment.get_template(posix_path) + content = template.render(**vars).strip() + content = content[:-1] if content.endswith(";") else content + return content - @staticmethod - def get_global_manager() -> 'SecretManager': - return SecretManager.__singleton + def relpath(self, file_path: str): + return os.path.relpath(file_path, self.__project_root) - @staticmethod - def set_global_manager(global_manager: 'SecretManager'): - SecretManager.__singleton = global_manager - @staticmethod - def global_redact(context: str) -> str: +class SecretManager: """ - redacts any text that has been classified a secret - using the global SecretManager instance. + Provides the ability to redact secrets """ - return SecretManager.__singleton.redact(context) - - def __init__(self): - self.__secrets = set() - def clear(self): - self.__secrets = set() + __singleton: "SecretManager" - def add(self, secret: str): - if secret: - self.__secrets.add(secret) + @staticmethod + def get_global_manager() -> "SecretManager": + return SecretManager.__singleton - def add_range(self, secrets: Set[str]): - if secrets: - self.__secrets = self.__secrets | secrets + @staticmethod + def set_global_manager(global_manager: "SecretManager"): + SecretManager.__singleton = global_manager - def redact(self, context: str) -> str: - """ - redacts any text that has been classified a secret - """ - redacted = context - if redacted: - for secret in self.__secrets: - redacted = redacted.replace(secret, "*" * len(secret)) - return redacted + @staticmethod + def global_redact(context: str) -> str: + """ + redacts any text that has been classified a secret + using the global SecretManager instance. + """ + return SecretManager.__singleton.redact(context) + def __init__(self): + self.__secrets = set() -class SnowflakeSchemachangeSession: - """ - Manages Snowflake Interactions and authentication - """ - #region Query Templates - _q_ch_metadata = "SELECT CREATED, LAST_ALTERED FROM {database_name}.INFORMATION_SCHEMA.TABLES" \ - + " WHERE TABLE_SCHEMA = REPLACE('{schema_name}','\"','') AND TABLE_NAME = replace('{table_name}','\"','')" - _q_ch_schema_present = "SELECT COUNT(1) FROM {database_name}.INFORMATION_SCHEMA.SCHEMATA" \ - + " WHERE SCHEMA_NAME = REPLACE('{schema_name}','\"','')" - _q_ch_ddl_schema = "CREATE SCHEMA {schema_name}" - _q_ch_ddl_table = "CREATE TABLE IF NOT EXISTS {database_name}.{schema_name}.{table_name} (VERSION VARCHAR, " \ - + "DESCRIPTION VARCHAR, SCRIPT VARCHAR, SCRIPT_TYPE VARCHAR, CHECKSUM VARCHAR," \ - + " EXECUTION_TIME NUMBER, STATUS VARCHAR, INSTALLED_BY VARCHAR, INSTALLED_ON TIMESTAMP_LTZ)" - _q_ch_r_checksum = "SELECT DISTINCT SCRIPT, FIRST_VALUE(CHECKSUM) OVER (PARTITION BY SCRIPT " \ - + "ORDER BY INSTALLED_ON DESC) FROM {database_name}.{schema_name}.{table_name} WHERE SCRIPT_TYPE = 'R' AND " \ - + "STATUS = 'Success'" - _q_ch_fetch ="SELECT VERSION FROM {database_name}.{schema_name}.{table_name} WHERE SCRIPT_TYPE = 'V' ORDER" \ - + " BY INSTALLED_ON DESC LIMIT 1" - _q_sess_tag = "ALTER SESSION SET QUERY_TAG = '{query_tag}'" - _q_ch_log = "INSERT INTO {database_name}.{schema_name}.{table_name} (VERSION, DESCRIPTION, SCRIPT, SCRIPT_TYPE, " \ - + "CHECKSUM, EXECUTION_TIME, STATUS, INSTALLED_BY, INSTALLED_ON) values ('{script_version}'," \ - + "'{script_description}','{script_name}','{script_type}','{checksum}',{execution_time}," \ - + "'{status}','{user}',CURRENT_TIMESTAMP);" - _q_set_sess_role = 'USE ROLE {role};' - _q_set_sess_database = 'USE DATABASE {database};' - _q_set_sess_schema = 'USE SCHEMA {schema};' - _q_set_sess_warehouse = 'USE WAREHOUSE {warehouse};' - #endregion Query Templates - - - def __init__(self, config): - session_parameters = { - "QUERY_TAG": "schemachange %s" % _schemachange_version - } - if config['query_tag']: - session_parameters["QUERY_TAG"] += ";%s" % config['query_tag'] - - # Retreive Connection info from config dictionary - self.conArgs = {"user": config['snowflake_user'],"account": config['snowflake_account'] \ - ,"role": config['snowflake_role'],"warehouse": config['snowflake_warehouse'] \ - ,"database": config['snowflake_database'],"schema": config['snowflake_schema'], "application": _snowflake_application_name \ - ,"session_parameters":session_parameters} - - self.oauth_config = config['oauth_config'] - self.autocommit = config['autocommit'] - self.verbose = config['verbose'] - if self.set_connection_args(): - self.con = snowflake.connector.connect(**self.conArgs) - if not self.autocommit: - self.con.autocommit(False) - else: - print(_err_env_missing) + def clear(self): + self.__secrets = set() - def __del__(self): - if hasattr(self, 'con'): - self.con.close() + def add(self, secret: str): + if secret: + self.__secrets.add(secret) - def get_oauth_token(self): - req_info = { \ - "url":self.oauth_config['token-provider-url'], \ - "headers":self.oauth_config['token-request-headers'], \ - "data":self.oauth_config['token-request-payload'] \ - } - token_name = self.oauth_config['token-response-name'] - response = requests.post(**req_info) - resJsonDict =json.loads(response.text) - try: - return resJsonDict[token_name] - except KeyError: - errormessage = _err_oauth_tk_nm.format( - keys = ', '.join(resJsonDict.keys()), - key = token_name - ) - # if there is an error passed with the reponse include that - if 'error_description' in resJsonDict.keys(): - errormessage += _err_oauth_tk_err.format(desc=resJsonDict['error_description']) - raise KeyError( errormessage ) - - def set_connection_args(self): - # Password authentication is the default - snowflake_password = None - default_authenticator = 'snowflake' - if os.getenv("SNOWFLAKE_PASSWORD") is not None and os.getenv("SNOWFLAKE_PASSWORD"): - snowflake_password = os.getenv("SNOWFLAKE_PASSWORD") - - # Check legacy/deprecated env variable - if os.getenv("SNOWSQL_PWD") is not None and os.getenv("SNOWSQL_PWD"): - if snowflake_password: - warnings.warn(_warn_password_dup, DeprecationWarning) - else: - warnings.warn(_warn_password, DeprecationWarning) - snowflake_password = os.getenv("SNOWSQL_PWD") - - snowflake_authenticator = os.getenv("SNOWFLAKE_AUTHENTICATOR") - - if snowflake_authenticator: - # Determine the type of Authenticator - # OAuth based authentication - if snowflake_authenticator.lower() == 'oauth': - oauth_token = self.get_oauth_token() + def add_range(self, secrets: Set[str]): + if secrets: + self.__secrets = self.__secrets | secrets - if self.verbose: - print( _log_auth_type % 'Oauth Access Token') - self.conArgs['token'] = oauth_token - self.conArgs['authenticator'] = 'oauth' - # External Browswer based SSO - elif snowflake_authenticator.lower() == 'externalbrowser': - self.conArgs['authenticator'] = 'externalbrowser' - if self.verbose: - print(_log_auth_type % 'External Browser') - # IDP based Authentication, limited to Okta - elif snowflake_authenticator.lower()[:8]=='https://': + def redact(self, context: str) -> str: + """ + redacts any text that has been classified a secret + """ + redacted = context + if redacted: + for secret in self.__secrets: + redacted = redacted.replace(secret, "*" * len(secret)) + return redacted - if self.verbose: - print(_log_auth_type % 'Okta') - print(_log_okta_ep % snowflake_authenticator) - - self.conArgs['password'] = snowflake_password - self.conArgs['authenticator'] = snowflake_authenticator.lower() - elif snowflake_authenticator.lower() == 'snowflake': - self.conArgs['authenticator'] = default_authenticator - # if authenticator is not a supported method or the authenticator variable is defined but not specified - else: - # defaulting to snowflake as authenticator - if self.verbose: - print(_err_unsupported_auth_mthd.format(unsupported_authenticator=snowflake_authenticator) ) - self.conArgs['authenticator'] = default_authenticator - else: - # default authenticator to snowflake - self.conArgs['authenticator'] = default_authenticator - if self.conArgs['authenticator'].lower() == default_authenticator: - # Giving preference to password based authentication when both private key and password are specified. - if snowflake_password: - if self.verbose: - print(_log_auth_type % 'password' ) - self.conArgs['password'] = snowflake_password +class SnowflakeSchemachangeSession: + """ + Manages Snowflake Interactions and authentication + """ - elif os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH", ''): - if self.verbose: - print( _log_auth_type % 'private key') + # region Query Templates + _q_ch_metadata = ( + "SELECT CREATED, LAST_ALTERED FROM {database_name}.INFORMATION_SCHEMA.TABLES" + + " WHERE TABLE_SCHEMA = REPLACE('{schema_name}','\"','') AND TABLE_NAME = replace('{table_name}','\"','')" + ) + _q_ch_schema_present = ( + "SELECT COUNT(1) FROM {database_name}.INFORMATION_SCHEMA.SCHEMATA" + + " WHERE SCHEMA_NAME = REPLACE('{schema_name}','\"','')" + ) + _q_ch_ddl_schema = "CREATE SCHEMA {schema_name}" + _q_ch_ddl_table = ( + "CREATE TABLE IF NOT EXISTS {database_name}.{schema_name}.{table_name} (VERSION VARCHAR, " + + "DESCRIPTION VARCHAR, SCRIPT VARCHAR, SCRIPT_TYPE VARCHAR, CHECKSUM VARCHAR," + + " EXECUTION_TIME NUMBER, STATUS VARCHAR, INSTALLED_BY VARCHAR, INSTALLED_ON TIMESTAMP_LTZ)" + ) + _q_ch_r_checksum = ( + "SELECT DISTINCT SCRIPT, FIRST_VALUE(CHECKSUM) OVER (PARTITION BY SCRIPT " + + "ORDER BY INSTALLED_ON DESC) FROM {database_name}.{schema_name}.{table_name} WHERE SCRIPT_TYPE = 'R' AND " + + "STATUS = 'Success'" + ) + _q_ch_fetch = ( + "SELECT VERSION FROM {database_name}.{schema_name}.{table_name} WHERE SCRIPT_TYPE = 'V' ORDER" + + " BY INSTALLED_ON DESC LIMIT 1" + ) + _q_sess_tag = "ALTER SESSION SET QUERY_TAG = '{query_tag}'" + _q_ch_log = ( + "INSERT INTO {database_name}.{schema_name}.{table_name} (VERSION, DESCRIPTION, SCRIPT, SCRIPT_TYPE, " + + "CHECKSUM, EXECUTION_TIME, STATUS, INSTALLED_BY, INSTALLED_ON) values ('{script_version}'," + + "'{script_description}','{script_name}','{script_type}','{checksum}',{execution_time}," + + "'{status}','{user}',CURRENT_TIMESTAMP);" + ) + _q_set_sess_role = "USE ROLE {role};" + _q_set_sess_database = "USE DATABASE {database};" + _q_set_sess_schema = "USE SCHEMA {schema};" + _q_set_sess_warehouse = "USE WAREHOUSE {warehouse};" + # endregion Query Templates + + def __init__(self, config): + session_parameters = {"QUERY_TAG": "schemachange %s" % _schemachange_version} + if config["query_tag"]: + session_parameters["QUERY_TAG"] += ";%s" % config["query_tag"] + + # Retreive Connection info from config dictionary + self.conArgs = { + "user": config["snowflake_user"], + "account": config["snowflake_account"], + "role": config["snowflake_role"], + "warehouse": config["snowflake_warehouse"], + "database": config["snowflake_database"], + "schema": config["snowflake_schema"], + "application": _snowflake_application_name, + "session_parameters": session_parameters, + } - private_key_password = os.getenv("SNOWFLAKE_PRIVATE_KEY_PASSPHRASE", '') - if private_key_password: - private_key_password = private_key_password.encode() + self.oauth_config = config["oauth_config"] + self.autocommit = config["autocommit"] + self.verbose = config["verbose"] + if self.set_connection_args(): + self.con = snowflake.connector.connect(**self.conArgs) + if not self.autocommit: + self.con.autocommit(False) else: - private_key_password = None - if self.verbose: - print(_log_pk_enc) - with open(os.environ["SNOWFLAKE_PRIVATE_KEY_PATH"], "rb") as key: - p_key= serialization.load_pem_private_key( - key.read(), - password = private_key_password, - backend = default_backend() - ) - - pkb = p_key.private_bytes( - encoding = serialization.Encoding.DER, - format = serialization.PrivateFormat.PKCS8, - encryption_algorithm = serialization.NoEncryption()) - - self.conArgs['private_key'] = pkb - else: - raise NameError(_err_no_auth_mthd) - - return True - - def execute_snowflake_query(self, query): - if self.verbose: - print(SecretManager.global_redact("SQL query: %s" % query)) - try: - res = self.con.execute_string(query) - if not self.autocommit: - self.con.commit() - return res - except Exception as e: - if not self.autocommit: - self.con.rollback() - raise e - - def fetch_change_history_metadata(self,change_history_table): - # This should only ever return 0 or 1 rows - query = self._q_ch_metadata.format(**change_history_table) - results = self.execute_snowflake_query(query) - - # Collect all the results into a list - change_history_metadata = dict() - for cursor in results: - for row in cursor: - change_history_metadata['created'] = row[0] - change_history_metadata['last_altered'] = row[1] - - return change_history_metadata - - def create_change_history_table_if_missing(self, change_history_table): - # Check if schema exists - query = self._q_ch_schema_present.format(**change_history_table) - results = self.execute_snowflake_query(query) - schema_exists = False - for cursor in results: - for row in cursor: - schema_exists = (row[0] > 0) - - # Create the schema if it doesn't exist - if not schema_exists: - query = self._q_ch_ddl_schema.format(**change_history_table) - self.execute_snowflake_query(query) - - # Finally, create the change history table if it doesn't exist - query = self._q_ch_ddl_table.format(**change_history_table) - self.execute_snowflake_query(query) - - def fetch_r_scripts_checksum(self,change_history_table): - query = self._q_ch_r_checksum.format(**change_history_table) - results = self.execute_snowflake_query(query) - - # Collect all the results into a dict - d_script_checksum = DataFrame(columns=['script_name', 'checksum']) - script_names = [] - checksums = [] - for cursor in results: - for row in cursor: - script_names.append(row[0]) - checksums.append(row[1]) - - d_script_checksum['script_name'] = script_names - d_script_checksum['checksum'] = checksums - return d_script_checksum - - def fetch_change_history(self, change_history_table): - query = self._q_ch_fetch.format(**change_history_table) - results = self.execute_snowflake_query(query) - - # Collect all the results into a list - change_history = list() - for cursor in results: - for row in cursor: - change_history.append(row[0]) - - return change_history - - def reset_session(self): - # These items are optional, so we can only reset the ones with values - reset_query = "" - if self.conArgs['role']: - reset_query += self._q_set_sess_role.format(**self.conArgs) + " " - if self.conArgs['warehouse']: - reset_query += self._q_set_sess_warehouse.format(**self.conArgs) + " " - if self.conArgs['database']: - reset_query += self._q_set_sess_database.format(**self.conArgs) + " " - if self.conArgs['schema']: - reset_query += self._q_set_sess_schema.format(**self.conArgs) + " " - - self.execute_snowflake_query(reset_query) - - def reset_query_tag(self, extra_tag = None): - query_tag = self.conArgs["session_parameters"]["QUERY_TAG"] - if extra_tag: - query_tag += f";{extra_tag}" - - self.execute_snowflake_query(self._q_sess_tag.format(query_tag=query_tag)) - - def apply_change_script(self, script, script_content, change_history_table): - # Define a few other change related variables - checksum = hashlib.sha224(script_content.encode('utf-8')).hexdigest() - execution_time = 0 - status = 'Success' - - # Execute the contents of the script - if len(script_content) > 0: - start = time.time() - self.reset_session() - self.reset_query_tag(script['script_name']) - self.execute_snowflake_query(script_content) - self.reset_query_tag() - self.reset_session() - end = time.time() - execution_time = round(end - start) - - # Finally record this change in the change history table by gathering data - frmt_args = script.copy() - frmt_args.update(change_history_table) - frmt_args['checksum'] =checksum - frmt_args['execution_time'] =execution_time - frmt_args['status'] =status - frmt_args['user'] =self.conArgs['user'] - # Compose and execute the insert statement to the log file - query = self._q_ch_log.format(**frmt_args) - self.execute_snowflake_query(query) + print(_err_env_missing) + def __del__(self): + if hasattr(self, "con"): + self.con.close() -def deploy_command(config): - # Make sure we have the required connection info, all of the below needs to be present. - req_args = {'snowflake_account','snowflake_user','snowflake_role','snowflake_warehouse'} - provided_args = {k:v for (k,v) in config.items() if v} - missing_args = req_args -provided_args.keys() - if len(missing_args)>0: - raise ValueError(_err_args_missing % ', '.join({s.replace('_', ' ') for s in missing_args})) - - #ensure an authentication method is specified / present. one of the below needs to be present. - req_env_var = {'SNOWFLAKE_PASSWORD', 'SNOWSQL_PWD','SNOWFLAKE_PRIVATE_KEY_PATH','SNOWFLAKE_AUTHENTICATOR'} - if len(req_env_var - dict(os.environ).keys())==len(req_env_var): - raise ValueError(_err_env_missing) - - # Log some additional details - if config['dry_run']: - print("Running in dry-run mode") - print(_log_config_details.format(**config)) - - #connect to snowflake and maintain connection - session = SnowflakeSchemachangeSession(config) - - scripts_skipped = 0 - scripts_applied = 0 - - # Deal with the change history table (create if specified) - change_history_table = get_change_history_table_details(config['change_history_table']) - change_history_metadata = session.fetch_change_history_metadata(change_history_table) - if change_history_metadata: - print(_log_ch_use.format(last_altered=change_history_metadata['last_altered'], **change_history_table)) - elif config['create_change_history_table']: - # Create the change history table (and containing objects) if it don't exist. - if not config['dry_run']: - session.create_change_history_table_if_missing(change_history_table) - print(_log_ch_create.format(**change_history_table)) - else: - raise ValueError(_err_ch_missing.format(**change_history_table)) - - # Find the max published version - max_published_version = '' - - change_history = None - r_scripts_checksum = None - if (config['dry_run'] and change_history_metadata) or not config['dry_run']: - change_history = session.fetch_change_history(change_history_table) - r_scripts_checksum = session.fetch_r_scripts_checksum(change_history_table) - - if change_history: - max_published_version = change_history[0] - max_published_version_display = max_published_version - if max_published_version_display == '': - max_published_version_display = 'None' - print(_log_ch_max_version.format(max_published_version_display=max_published_version_display)) - - # Find all scripts in the root folder (recursively) and sort them correctly - all_scripts = get_all_scripts_recursively(config['root_folder'], config['verbose']) - all_script_names = list(all_scripts.keys()) - # Sort scripts such that versioned scripts get applied first and then the repeatable ones. - all_script_names_sorted = sorted_alphanumeric([script for script in all_script_names if script[0] == 'V']) \ - + sorted_alphanumeric([script for script in all_script_names if script[0] == 'R']) \ - + sorted_alphanumeric([script for script in all_script_names if script[0] == 'A']) - - # Loop through each script in order and apply any required changes - for script_name in all_script_names_sorted: - script = all_scripts[script_name] - - # Apply a versioned-change script only if the version is newer than the most recent change in the database - # Apply any other scripts, i.e. repeatable scripts, irrespective of the most recent change in the database - if script_name[0] == 'V' and get_alphanum_key(script['script_version']) <= get_alphanum_key(max_published_version): - if config['verbose']: - print(_log_skip_v.format(max_published_version=max_published_version, **script) ) - scripts_skipped += 1 - continue - - # Always process with jinja engine - jinja_processor = JinjaTemplateProcessor(project_root = config['root_folder'], modules_folder = config['modules_folder']) - content = jinja_processor.render(jinja_processor.relpath(script['script_full_path']), config['vars'], config['verbose']) - - # Apply only R scripts where the checksum changed compared to the last execution of snowchange - if script_name[0] == 'R': - # Compute the checksum for the script - checksum_current = hashlib.sha224(content.encode('utf-8')).hexdigest() + def get_oauth_token(self): + req_info = { + "url": self.oauth_config["token-provider-url"], + "headers": self.oauth_config["token-request-headers"], + "data": self.oauth_config["token-request-payload"], + } + token_name = self.oauth_config["token-response-name"] + response = requests.post(**req_info) + resJsonDict = json.loads(response.text) + try: + return resJsonDict[token_name] + except KeyError: + errormessage = _err_oauth_tk_nm.format( + keys=", ".join(resJsonDict.keys()), key=token_name + ) + # if there is an error passed with the reponse include that + if "error_description" in resJsonDict.keys(): + errormessage += _err_oauth_tk_err.format( + desc=resJsonDict["error_description"] + ) + raise KeyError(errormessage) + + def set_connection_args(self): + # Password authentication is the default + snowflake_password = None + default_authenticator = "snowflake" + if os.getenv("SNOWFLAKE_PASSWORD") is not None and os.getenv( + "SNOWFLAKE_PASSWORD" + ): + snowflake_password = os.getenv("SNOWFLAKE_PASSWORD") + + # Check legacy/deprecated env variable + if os.getenv("SNOWSQL_PWD") is not None and os.getenv("SNOWSQL_PWD"): + if snowflake_password: + warnings.warn(_warn_password_dup, DeprecationWarning) + else: + warnings.warn(_warn_password, DeprecationWarning) + snowflake_password = os.getenv("SNOWSQL_PWD") + + snowflake_authenticator = os.getenv("SNOWFLAKE_AUTHENTICATOR") + + if snowflake_authenticator: + # Determine the type of Authenticator + # OAuth based authentication + if snowflake_authenticator.lower() == "oauth": + oauth_token = self.get_oauth_token() + + if self.verbose: + print(_log_auth_type % "Oauth Access Token") + self.conArgs["token"] = oauth_token + self.conArgs["authenticator"] = "oauth" + # External Browswer based SSO + elif snowflake_authenticator.lower() == "externalbrowser": + self.conArgs["authenticator"] = "externalbrowser" + if self.verbose: + print(_log_auth_type % "External Browser") + # IDP based Authentication, limited to Okta + elif snowflake_authenticator.lower()[:8] == "https://": + + if self.verbose: + print(_log_auth_type % "Okta") + print(_log_okta_ep % snowflake_authenticator) + + self.conArgs["password"] = snowflake_password + self.conArgs["authenticator"] = snowflake_authenticator.lower() + elif snowflake_authenticator.lower() == "snowflake": + self.conArgs["authenticator"] = default_authenticator + # if authenticator is not a supported method or the authenticator variable is defined but not specified + else: + # defaulting to snowflake as authenticator + if self.verbose: + print( + _err_unsupported_auth_mthd.format( + unsupported_authenticator=snowflake_authenticator + ) + ) + self.conArgs["authenticator"] = default_authenticator + else: + # default authenticator to snowflake + self.conArgs["authenticator"] = default_authenticator + + if self.conArgs["authenticator"].lower() == default_authenticator: + # Giving preference to password based authentication when both private key and password are specified. + if snowflake_password: + if self.verbose: + print(_log_auth_type % "password") + self.conArgs["password"] = snowflake_password + + elif os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH", ""): + if self.verbose: + print(_log_auth_type % "private key") + + private_key_password = os.getenv("SNOWFLAKE_PRIVATE_KEY_PASSPHRASE", "") + if private_key_password: + private_key_password = private_key_password.encode() + else: + private_key_password = None + if self.verbose: + print(_log_pk_enc) + with open(os.environ["SNOWFLAKE_PRIVATE_KEY_PATH"], "rb") as key: + p_key = serialization.load_pem_private_key( + key.read(), + password=private_key_password, + backend=default_backend(), + ) + + pkb = p_key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + self.conArgs["private_key"] = pkb + else: + raise NameError(_err_no_auth_mthd) + + return True + + def execute_snowflake_query(self, query): + if self.verbose: + print(SecretManager.global_redact("SQL query: %s" % query)) + try: + res = self.con.execute_string(query) + if not self.autocommit: + self.con.commit() + return res + except Exception as e: + if not self.autocommit: + self.con.rollback() + raise e + + def fetch_change_history_metadata(self, change_history_table): + # This should only ever return 0 or 1 rows + query = self._q_ch_metadata.format(**change_history_table) + results = self.execute_snowflake_query(query) + + # Collect all the results into a list + change_history_metadata = dict() + for cursor in results: + for row in cursor: + change_history_metadata["created"] = row[0] + change_history_metadata["last_altered"] = row[1] + + return change_history_metadata + + def create_change_history_table_if_missing(self, change_history_table): + # Check if schema exists + query = self._q_ch_schema_present.format(**change_history_table) + results = self.execute_snowflake_query(query) + schema_exists = False + for cursor in results: + for row in cursor: + schema_exists = row[0] > 0 + + # Create the schema if it doesn't exist + if not schema_exists: + query = self._q_ch_ddl_schema.format(**change_history_table) + self.execute_snowflake_query(query) + + # Finally, create the change history table if it doesn't exist + query = self._q_ch_ddl_table.format(**change_history_table) + self.execute_snowflake_query(query) + + def fetch_r_scripts_checksum(self, change_history_table): + query = self._q_ch_r_checksum.format(**change_history_table) + results = self.execute_snowflake_query(query) + + # Collect all the results into a dict + d_script_checksum = DataFrame(columns=["script_name", "checksum"]) + script_names = [] + checksums = [] + for cursor in results: + for row in cursor: + script_names.append(row[0]) + checksums.append(row[1]) + + d_script_checksum["script_name"] = script_names + d_script_checksum["checksum"] = checksums + return d_script_checksum + + def fetch_change_history(self, change_history_table): + query = self._q_ch_fetch.format(**change_history_table) + results = self.execute_snowflake_query(query) + + # Collect all the results into a list + change_history = list() + for cursor in results: + for row in cursor: + change_history.append(row[0]) + + return change_history + + def reset_session(self): + # These items are optional, so we can only reset the ones with values + reset_query = "" + if self.conArgs["role"]: + reset_query += self._q_set_sess_role.format(**self.conArgs) + " " + if self.conArgs["warehouse"]: + reset_query += self._q_set_sess_warehouse.format(**self.conArgs) + " " + if self.conArgs["database"]: + reset_query += self._q_set_sess_database.format(**self.conArgs) + " " + if self.conArgs["schema"]: + reset_query += self._q_set_sess_schema.format(**self.conArgs) + " " + + self.execute_snowflake_query(reset_query) + + def reset_query_tag(self, extra_tag=None): + query_tag = self.conArgs["session_parameters"]["QUERY_TAG"] + if extra_tag: + query_tag += f";{extra_tag}" + + self.execute_snowflake_query(self._q_sess_tag.format(query_tag=query_tag)) + + def apply_change_script(self, script, script_content, change_history_table): + # Define a few other change related variables + checksum = hashlib.sha224(script_content.encode("utf-8")).hexdigest() + execution_time = 0 + status = "Success" + + # Execute the contents of the script + if len(script_content) > 0: + start = time.time() + self.reset_session() + self.reset_query_tag(script["script_name"]) + self.execute_snowflake_query(script_content) + self.reset_query_tag() + self.reset_session() + end = time.time() + execution_time = round(end - start) + + # Finally record this change in the change history table by gathering data + frmt_args = script.copy() + frmt_args.update(change_history_table) + frmt_args["checksum"] = checksum + frmt_args["execution_time"] = execution_time + frmt_args["status"] = status + frmt_args["user"] = self.conArgs["user"] + # Compose and execute the insert statement to the log file + query = self._q_ch_log.format(**frmt_args) + self.execute_snowflake_query(query) - # check if R file was already executed - if (r_scripts_checksum is not None) and script_name in list(r_scripts_checksum['script_name']): - checksum_last = list(r_scripts_checksum.loc[r_scripts_checksum['script_name'] == script_name, 'checksum'])[0] - else: - checksum_last = '' - # check if there is a change of the checksum in the script - if checksum_current == checksum_last: - if config['verbose']: - print(_log_skip_r.format(**script)) - scripts_skipped += 1 - continue +def deploy_command(config): + # Make sure we have the required connection info, all of the below needs to be present. + req_args = { + "snowflake_account", + "snowflake_user", + "snowflake_role", + "snowflake_warehouse", + } + provided_args = {k: v for (k, v) in config.items() if v} + missing_args = req_args - provided_args.keys() + if len(missing_args) > 0: + raise ValueError( + _err_args_missing % ", ".join({s.replace("_", " ") for s in missing_args}) + ) + + # ensure an authentication method is specified / present. one of the below needs to be present. + req_env_var = { + "SNOWFLAKE_PASSWORD", + "SNOWSQL_PWD", + "SNOWFLAKE_PRIVATE_KEY_PATH", + "SNOWFLAKE_AUTHENTICATOR", + } + if len(req_env_var - dict(os.environ).keys()) == len(req_env_var): + raise ValueError(_err_env_missing) + + # Log some additional details + if config["dry_run"]: + print("Running in dry-run mode") + print(_log_config_details.format(**config)) + + # connect to snowflake and maintain connection + session = SnowflakeSchemachangeSession(config) + + scripts_skipped = 0 + scripts_applied = 0 + + # Deal with the change history table (create if specified) + change_history_table = get_change_history_table_details( + config["change_history_table"] + ) + change_history_metadata = session.fetch_change_history_metadata( + change_history_table + ) + if change_history_metadata: + print( + _log_ch_use.format( + last_altered=change_history_metadata["last_altered"], + **change_history_table, + ) + ) + elif config["create_change_history_table"]: + # Create the change history table (and containing objects) if it don't exist. + if not config["dry_run"]: + session.create_change_history_table_if_missing(change_history_table) + print(_log_ch_create.format(**change_history_table)) + else: + raise ValueError(_err_ch_missing.format(**change_history_table)) + + # Find the max published version + max_published_version = "" + + change_history = None + r_scripts_checksum = None + if (config["dry_run"] and change_history_metadata) or not config["dry_run"]: + change_history = session.fetch_change_history(change_history_table) + r_scripts_checksum = session.fetch_r_scripts_checksum(change_history_table) + + if change_history: + max_published_version = change_history[0] + max_published_version_display = max_published_version + if max_published_version_display == "": + max_published_version_display = "None" + print( + _log_ch_max_version.format( + max_published_version_display=max_published_version_display + ) + ) + + # Find all scripts in the root folder (recursively) and sort them correctly + all_scripts = get_all_scripts_recursively(config["root_folder"], config["verbose"]) + all_script_names = list(all_scripts.keys()) + # Sort scripts such that versioned scripts get applied first and then the repeatable ones. + all_script_names_sorted = ( + sorted_alphanumeric([script for script in all_script_names if script[0] == "V"]) + + sorted_alphanumeric( + [script for script in all_script_names if script[0] == "R"] + ) + + sorted_alphanumeric( + [script for script in all_script_names if script[0] == "A"] + ) + ) + + # Loop through each script in order and apply any required changes + for script_name in all_script_names_sorted: + script = all_scripts[script_name] + + # Apply a versioned-change script only if the version is newer than the most recent change in the database + # Apply any other scripts, i.e. repeatable scripts, irrespective of the most recent change in the database + if script_name[0] == "V" and get_alphanum_key( + script["script_version"] + ) <= get_alphanum_key(max_published_version): + if config["verbose"]: + print( + _log_skip_v.format( + max_published_version=max_published_version, **script + ) + ) + scripts_skipped += 1 + continue + + # Always process with jinja engine + jinja_processor = JinjaTemplateProcessor( + project_root=config["root_folder"], modules_folder=config["modules_folder"] + ) + content = jinja_processor.render( + jinja_processor.relpath(script["script_full_path"]), + config["vars"], + config["verbose"], + ) + + # Apply only R scripts where the checksum changed compared to the last execution of snowchange + if script_name[0] == "R": + # Compute the checksum for the script + checksum_current = hashlib.sha224(content.encode("utf-8")).hexdigest() + + # check if R file was already executed + if (r_scripts_checksum is not None) and script_name in list( + r_scripts_checksum["script_name"] + ): + checksum_last = list( + r_scripts_checksum.loc[ + r_scripts_checksum["script_name"] == script_name, "checksum" + ] + )[0] + else: + checksum_last = "" + + # check if there is a change of the checksum in the script + if checksum_current == checksum_last: + if config["verbose"]: + print(_log_skip_r.format(**script)) + scripts_skipped += 1 + continue + + print(_log_apply.format(**script)) + if not config["dry_run"]: + session.apply_change_script(script, content, change_history_table) + + scripts_applied += 1 + + print( + _log_apply_set_complete.format( + scripts_applied=scripts_applied, scripts_skipped=scripts_skipped + ) + ) - print(_log_apply.format(**script)) - if not config['dry_run']: - session.apply_change_script(script, content, change_history_table) - scripts_applied += 1 +def render_command(config, script_path): + """ + Renders the provided script. - print(_log_apply_set_complete.format(scripts_applied=scripts_applied, scripts_skipped=scripts_skipped)) + Note: does not apply secrets filtering. + """ + # Validate the script file path + script_path = os.path.abspath(script_path) + if not os.path.isfile(script_path): + raise ValueError( + _err_invalid_folder.format(folder_type="script_path", path=script_path) + ) + # Always process with jinja engine + jinja_processor = JinjaTemplateProcessor( + project_root=config["root_folder"], modules_folder=config["modules_folder"] + ) + content = jinja_processor.render( + jinja_processor.relpath(script_path), config["vars"], config["verbose"] + ) + + checksum = hashlib.sha224(content.encode("utf-8")).hexdigest() + print("Checksum %s" % checksum) + print(content) + + +def alphanum_convert(text: str): + result = None + if text.isdigit(): + result = int(text) + else: + result = text.lower() + return result -def render_command(config, script_path): - """ - Renders the provided script. - - Note: does not apply secrets filtering. - """ - # Validate the script file path - script_path = os.path.abspath(script_path) - if not os.path.isfile(script_path): - raise ValueError(_err_invalid_folder.format(folder_type='script_path', path=script_path)) - # Always process with jinja engine - jinja_processor = JinjaTemplateProcessor(project_root = config['root_folder'], \ - modules_folder = config['modules_folder']) - content = jinja_processor.render(jinja_processor.relpath(script_path), \ - config['vars'], config['verbose']) - - checksum = hashlib.sha224(content.encode('utf-8')).hexdigest() - print("Checksum %s" % checksum) - print(content) - -def alphanum_convert(text:str): - result = None - if text.isdigit(): - result = int(text) - else: - result = text.lower() - return result # This function will return a list containing the parts of the key (split by number parts) # Each number is converted to and integer and string parts are left as strings # This will enable correct sorting in python when the lists are compared # e.g. get_alphanum_key('1.2.2') results in ['', 1, '.', 2, '.', 2, ''] def get_alphanum_key(key): - alphanum_key = [ alphanum_convert(c) for c in re.split('([0-9]+)', key) ] - return alphanum_key + alphanum_key = [alphanum_convert(c) for c in re.split("([0-9]+)", key)] + return alphanum_key + def sorted_alphanumeric(data): - return sorted(data, key=get_alphanum_key) + return sorted(data, key=get_alphanum_key) + def load_schemachange_config(config_file_path: str) -> Dict[str, Any]: - """ - Loads the schemachange config file and processes with jinja templating engine - """ - config = dict() - - # First read in the yaml config file, if present - if os.path.isfile(config_file_path): - with open(config_file_path) as config_file: - # Run the config file through the jinja engine to give access to environmental variables - # The config file does not have the same access to the jinja functionality that a script - # has. - config_template = jinja2.Template(config_file.read(), undefined=jinja2.StrictUndefined, extensions=[JinjaEnvVar]) - - # The FullLoader parameter handles the conversion from YAML scalar values to Python the dictionary format - config = yaml.load(config_template.render(), Loader=yaml.FullLoader) - print("Using config file: %s" % config_file_path) - return config - -def get_schemachange_config(config_file_path, root_folder, modules_folder, snowflake_account, \ - snowflake_user, snowflake_role, snowflake_warehouse, snowflake_database, snowflake_schema, \ - change_history_table, vars, create_change_history_table, autocommit, verbose, \ - dry_run, query_tag, oauth_config, **kwargs): - - # create cli override dictionary - # Could refactor to just pass Args as a dictionary? - # **kwargs inlcuded to avoid complaints about unexpect arguments from arg parser eg:subcommand - cli_inputs = { "root_folder":root_folder, \ - "modules_folder":modules_folder, "snowflake_account":snowflake_account, \ - "snowflake_user":snowflake_user, "snowflake_role":snowflake_role, \ - "snowflake_warehouse":snowflake_warehouse, "snowflake_database":snowflake_database, \ - "snowflake_schema":snowflake_schema, \ - "change_history_table":change_history_table, "vars":vars, \ - "create_change_history_table":create_change_history_table, \ - "autocommit":autocommit, "verbose":verbose, "dry_run":dry_run,\ - "query_tag":query_tag, "oauth_config":oauth_config} - cli_inputs = {k:v for (k,v) in cli_inputs.items() if v} - - # load YAML inputs and convert kebabs to snakes - config = {k.replace('-','_'):v for (k,v) in load_schemachange_config(config_file_path).items()} - # set values passed into the cli Overriding values in config file - config.update(cli_inputs) - - # create Default values dictionary - config_defaults = {"root_folder":os.path.abspath('.'), "modules_folder":None, \ - "snowflake_account":None, "snowflake_user":None, "snowflake_role":None, \ - "snowflake_warehouse":None, "snowflake_database":None, "snowflake_schema":None, \ - "change_history_table":None, \ - "vars":{}, "create_change_history_table":False, "autocommit":False, "verbose":False, \ - "dry_run":False , "query_tag":None , "oauth_config":None } - #insert defualt values for items not populated - config.update({ k:v for (k,v) in config_defaults.items() if k not in config.keys()}) - - # Validate folder paths - if 'root_folder' in config: - config['root_folder'] = os.path.abspath(config['root_folder']) - if not os.path.isdir(config['root_folder']): - raise ValueError(_err_invalid_folder.format(folder_type='root', path=config['root_folder'])) - - if config['modules_folder']: - config['modules_folder'] = os.path.abspath(config['modules_folder']) - if not os.path.isdir(config['modules_folder']): - raise ValueError(_err_invalid_folder.format(folder_type='modules', path=config['modules_folder'])) - if config['vars']: - # if vars is configured wrong in the config file it will come through as a string - if not isinstance(config['vars'], dict): - raise ValueError(_err_vars_config) - - # the variable schema change has been reserved - if "schemachange" in config['vars']: - raise ValueError(_err_vars_reserved) - - return config + """ + Loads the schemachange config file and processes with jinja templating engine + """ + config = dict() + + # First read in the yaml config file, if present + if os.path.isfile(config_file_path): + with open(config_file_path) as config_file: + # Run the config file through the jinja engine to give access to environmental variables + # The config file does not have the same access to the jinja functionality that a script + # has. + config_template = jinja2.Template( + config_file.read(), + undefined=jinja2.StrictUndefined, + extensions=[JinjaEnvVar], + ) + + # The FullLoader parameter handles the conversion from YAML scalar values to Python the dictionary format + config = yaml.load(config_template.render(), Loader=yaml.FullLoader) + print("Using config file: %s" % config_file_path) + return config + + +def get_schemachange_config( + config_file_path, + root_folder, + modules_folder, + snowflake_account, + snowflake_user, + snowflake_role, + snowflake_warehouse, + snowflake_database, + snowflake_schema, + change_history_table, + vars, + create_change_history_table, + autocommit, + verbose, + dry_run, + query_tag, + oauth_config, + **kwargs, +): + + # create cli override dictionary + # Could refactor to just pass Args as a dictionary? + # **kwargs inlcuded to avoid complaints about unexpect arguments from arg parser eg:subcommand + cli_inputs = { + "root_folder": root_folder, + "modules_folder": modules_folder, + "snowflake_account": snowflake_account, + "snowflake_user": snowflake_user, + "snowflake_role": snowflake_role, + "snowflake_warehouse": snowflake_warehouse, + "snowflake_database": snowflake_database, + "snowflake_schema": snowflake_schema, + "change_history_table": change_history_table, + "vars": vars, + "create_change_history_table": create_change_history_table, + "autocommit": autocommit, + "verbose": verbose, + "dry_run": dry_run, + "query_tag": query_tag, + "oauth_config": oauth_config, + } + cli_inputs = {k: v for (k, v) in cli_inputs.items() if v} + + # load YAML inputs and convert kebabs to snakes + config = { + k.replace("-", "_"): v + for (k, v) in load_schemachange_config(config_file_path).items() + } + # set values passed into the cli Overriding values in config file + config.update(cli_inputs) + + # create Default values dictionary + config_defaults = { + "root_folder": os.path.abspath("."), + "modules_folder": None, + "snowflake_account": None, + "snowflake_user": None, + "snowflake_role": None, + "snowflake_warehouse": None, + "snowflake_database": None, + "snowflake_schema": None, + "change_history_table": None, + "vars": {}, + "create_change_history_table": False, + "autocommit": False, + "verbose": False, + "dry_run": False, + "query_tag": None, + "oauth_config": None, + } + # insert defualt values for items not populated + config.update( + {k: v for (k, v) in config_defaults.items() if k not in config.keys()} + ) + + # Validate folder paths + if "root_folder" in config: + config["root_folder"] = os.path.abspath(config["root_folder"]) + if not os.path.isdir(config["root_folder"]): + raise ValueError( + _err_invalid_folder.format(folder_type="root", path=config["root_folder"]) + ) + + if config["modules_folder"]: + config["modules_folder"] = os.path.abspath(config["modules_folder"]) + if not os.path.isdir(config["modules_folder"]): + raise ValueError( + _err_invalid_folder.format( + folder_type="modules", path=config["modules_folder"] + ) + ) + if config["vars"]: + # if vars is configured wrong in the config file it will come through as a string + if not isinstance(config["vars"], dict): + raise ValueError(_err_vars_config) + + # the variable schema change has been reserved + if "schemachange" in config["vars"]: + raise ValueError(_err_vars_reserved) + + return config + def get_all_scripts_recursively(root_directory, verbose): - all_files = dict() - all_versions = list() - # Walk the entire directory structure recursively - for (directory_path, directory_names, file_names) in os.walk(root_directory): - for file_name in file_names: - - file_full_path = os.path.join(directory_path, file_name) - script_name_parts = re.search(r'^([V])(.+?)__(.+?)\.(?:sql|sql.jinja)$', \ - file_name.strip(), re.IGNORECASE) - repeatable_script_name_parts = re.search(r'^([R])__(.+?)\.(?:sql|sql.jinja)$', \ - file_name.strip(), re.IGNORECASE) - always_script_name_parts = re.search(r'^([A])__(.+?)\.(?:sql|sql.jinja)$', \ - file_name.strip(), re.IGNORECASE) - - # Set script type depending on whether it matches the versioned file naming format - if script_name_parts is not None: - script_type = 'V' - if verbose: - print("Found Versioned file " + file_full_path) - elif repeatable_script_name_parts is not None: - script_type = 'R' - if verbose: - print("Found Repeatable file " + file_full_path) - elif always_script_name_parts is not None: - script_type = 'A' - if verbose: - print("Found Always file " + file_full_path) - else: - if verbose: - print("Ignoring non-change file " + file_full_path) - continue - - # script name is the filename without any jinja extension - (file_part, extension_part) = os.path.splitext(file_name) - if extension_part.upper() == ".JINJA": - script_name = file_part - else: - script_name = file_name - - # Add this script to our dictionary (as nested dictionary) - script = dict() - script['script_name'] = script_name - script['script_full_path'] = file_full_path - script['script_type'] = script_type - script['script_version'] = '' if script_type in ['R', 'A'] else script_name_parts.group(2) - if script_type == 'R': - script['script_description'] = repeatable_script_name_parts.group(2).replace('_', ' ').capitalize() - elif script_type == 'A': - script['script_description'] = always_script_name_parts.group(2).replace('_', ' ').capitalize() - else: - script['script_description'] = script_name_parts.group(3).replace('_', ' ').capitalize() - - # Throw an error if the script_name already exists - if script_name in all_files: - raise ValueError( _err_dup_scripts.format(first_path=all_files[script_name]['script_full_path'], **script)) - - all_files[script_name] = script - - # Throw an error if the same version exists more than once - if script_type == 'V': - if script['script_version'] in all_versions: - raise ValueError(_err_dup_scripts_version.format(**script)) - all_versions.append(script['script_version']) - - return all_files + all_files = dict() + all_versions = list() + # Walk the entire directory structure recursively + for (directory_path, directory_names, file_names) in os.walk(root_directory): + for file_name in file_names: + + file_full_path = os.path.join(directory_path, file_name) + script_name_parts = re.search( + r"^([V])(.+?)__(.+?)\.(?:sql|sql.jinja)$", + file_name.strip(), + re.IGNORECASE, + ) + repeatable_script_name_parts = re.search( + r"^([R])__(.+?)\.(?:sql|sql.jinja)$", file_name.strip(), re.IGNORECASE + ) + always_script_name_parts = re.search( + r"^([A])__(.+?)\.(?:sql|sql.jinja)$", file_name.strip(), re.IGNORECASE + ) + + # Set script type depending on whether it matches the versioned file naming format + if script_name_parts is not None: + script_type = "V" + if verbose: + print("Found Versioned file " + file_full_path) + elif repeatable_script_name_parts is not None: + script_type = "R" + if verbose: + print("Found Repeatable file " + file_full_path) + elif always_script_name_parts is not None: + script_type = "A" + if verbose: + print("Found Always file " + file_full_path) + else: + if verbose: + print("Ignoring non-change file " + file_full_path) + continue + + # script name is the filename without any jinja extension + (file_part, extension_part) = os.path.splitext(file_name) + if extension_part.upper() == ".JINJA": + script_name = file_part + else: + script_name = file_name + + # Add this script to our dictionary (as nested dictionary) + script = dict() + script["script_name"] = script_name + script["script_full_path"] = file_full_path + script["script_type"] = script_type + script["script_version"] = ( + "" if script_type in ["R", "A"] else script_name_parts.group(2) + ) + if script_type == "R": + script["script_description"] = ( + repeatable_script_name_parts.group(2).replace("_", " ").capitalize() + ) + elif script_type == "A": + script["script_description"] = ( + always_script_name_parts.group(2).replace("_", " ").capitalize() + ) + else: + script["script_description"] = ( + script_name_parts.group(3).replace("_", " ").capitalize() + ) + + # Throw an error if the script_name already exists + if script_name in all_files: + raise ValueError( + _err_dup_scripts.format( + first_path=all_files[script_name]["script_full_path"], **script + ) + ) + + all_files[script_name] = script + + # Throw an error if the same version exists more than once + if script_type == "V": + if script["script_version"] in all_versions: + raise ValueError(_err_dup_scripts_version.format(**script)) + all_versions.append(script["script_version"]) + + return all_files + def get_change_history_table_details(change_history_table): - # Start with the global defaults - details = dict() - details['database_name'] = _metadata_database_name - details['schema_name'] = _metadata_schema_name - details['table_name'] = _metadata_table_name - - # Then override the defaults if requested. The name could be in one, two or three part notation. - if change_history_table is not None: - table_name_parts = change_history_table.strip().split('.') - if len(table_name_parts) == 1: - details['table_name'] = table_name_parts[0] - elif len(table_name_parts) == 2: - details['table_name'] = table_name_parts[1] - details['schema_name'] = table_name_parts[0] - elif len(table_name_parts) == 3: - details['table_name'] = table_name_parts[2] - details['schema_name'] = table_name_parts[1] - details['database_name'] = table_name_parts[0] - else: - raise ValueError(_err_invalid_cht % change_history_table) - #if the object name does not include '"' raise to upper case on return - return {k:v if '"' in v else v.upper() for (k,v) in details.items()} + # Start with the global defaults + details = dict() + details["database_name"] = _metadata_database_name + details["schema_name"] = _metadata_schema_name + details["table_name"] = _metadata_table_name + + # Then override the defaults if requested. The name could be in one, two or three part notation. + if change_history_table is not None: + table_name_parts = change_history_table.strip().split(".") + if len(table_name_parts) == 1: + details["table_name"] = table_name_parts[0] + elif len(table_name_parts) == 2: + details["table_name"] = table_name_parts[1] + details["schema_name"] = table_name_parts[0] + elif len(table_name_parts) == 3: + details["table_name"] = table_name_parts[2] + details["schema_name"] = table_name_parts[1] + details["database_name"] = table_name_parts[0] + else: + raise ValueError(_err_invalid_cht % change_history_table) + # if the object name does not include '"' raise to upper case on return + return {k: v if '"' in v else v.upper() for (k, v) in details.items()} -def extract_config_secrets(config: Dict[str, Any]) -> Set[str]: - """ - Extracts all secret values from the vars attributes in config - """ - # defined as an inner/ nested function to provide encapsulation - def inner_extract_dictionary_secrets(dictionary: Dict[str, Any], child_of_secrets: bool = False) -> Set[str]: +def extract_config_secrets(config: Dict[str, Any]) -> Set[str]: """ - Considers any key with the word secret in the name as a secret or - all values as secrets if a child of a key named secrets. + Extracts all secret values from the vars attributes in config """ - extracted_secrets: Set[str] = set() - - if dictionary: - for (key, value) in dictionary.items(): - if isinstance(value, dict): - if key == "secrets": - extracted_secrets = extracted_secrets | inner_extract_dictionary_secrets(value, True) - else : - extracted_secrets = extracted_secrets | inner_extract_dictionary_secrets(value, child_of_secrets) - elif child_of_secrets or "SECRET" in key.upper(): - extracted_secrets.add(value.strip()) - return extracted_secrets - - extracted = set() - - if config: - if "vars" in config: - extracted = inner_extract_dictionary_secrets(config["vars"]) - return extracted + + # defined as an inner/ nested function to provide encapsulation + def inner_extract_dictionary_secrets( + dictionary: Dict[str, Any], child_of_secrets: bool = False + ) -> Set[str]: + """ + Considers any key with the word secret in the name as a secret or + all values as secrets if a child of a key named secrets. + """ + extracted_secrets: Set[str] = set() + + if dictionary: + for (key, value) in dictionary.items(): + if isinstance(value, dict): + if key == "secrets": + extracted_secrets = ( + extracted_secrets + | inner_extract_dictionary_secrets(value, True) + ) + else: + extracted_secrets = ( + extracted_secrets + | inner_extract_dictionary_secrets(value, child_of_secrets) + ) + elif child_of_secrets or "SECRET" in key.upper(): + extracted_secrets.add(value.strip()) + return extracted_secrets + + extracted = set() + + if config: + if "vars" in config: + extracted = inner_extract_dictionary_secrets(config["vars"]) + return extracted + def main(argv=sys.argv): - parser = argparse.ArgumentParser(prog = 'schemachange', description = 'Apply schema changes to a Snowflake account. Full readme at https://github.com/Snowflake-Labs/schemachange', formatter_class = argparse.RawTextHelpFormatter) - subcommands = parser.add_subparsers(dest='subcommand') - - parser_deploy = subcommands.add_parser("deploy") - parser_deploy.add_argument('--config-folder', type = str, default = '.', help = 'The folder to look in for the schemachange-config.yml file (the default is the current working directory)', required = False) - parser_deploy.add_argument('-f', '--root-folder', type = str, help = 'The root folder for the database change scripts', required = False) - parser_deploy.add_argument('-m', '--modules-folder', type = str, help = 'The modules folder for jinja macros and templates to be used across multiple scripts', required = False) - parser_deploy.add_argument('-a', '--snowflake-account', type = str, help = 'The name of the snowflake account (e.g. xy12345.east-us-2.azure)', required = False) - parser_deploy.add_argument('-u', '--snowflake-user', type = str, help = 'The name of the snowflake user', required = False) - parser_deploy.add_argument('-r', '--snowflake-role', type = str, help = 'The name of the default role to use', required = False) - parser_deploy.add_argument('-w', '--snowflake-warehouse', type = str, help = 'The name of the default warehouse to use. Can be overridden in the change scripts.', required = False) - parser_deploy.add_argument('-d', '--snowflake-database', type = str, help = 'The name of the default database to use. Can be overridden in the change scripts.', required = False) - parser_deploy.add_argument('-s', '--snowflake-schema', type = str, help = 'The name of the default schema to use. Can be overridden in the change scripts.', required = False) - parser_deploy.add_argument('-c', '--change-history-table', type = str, help = 'Used to override the default name of the change history table (the default is METADATA.SCHEMACHANGE.CHANGE_HISTORY)', required = False) - parser_deploy.add_argument('--vars', type = json.loads, help = 'Define values for the variables to replaced in change scripts, given in JSON format (e.g. {"variable1": "value1", "variable2": "value2"})', required = False) - parser_deploy.add_argument('--create-change-history-table', action='store_true', help = 'Create the change history schema and table, if they do not exist (the default is False)', required = False) - parser_deploy.add_argument('-ac', '--autocommit', action='store_true', help = 'Enable autocommit feature for DML commands (the default is False)', required = False) - parser_deploy.add_argument('-v','--verbose', action='store_true', help = 'Display verbose debugging details during execution (the default is False)', required = False) - parser_deploy.add_argument('--dry-run', action='store_true', help = 'Run schemachange in dry run mode (the default is False)', required = False) - parser_deploy.add_argument('--query-tag', type = str, help = 'The string to add to the Snowflake QUERY_TAG session value for each query executed', required = False) - parser_deploy.add_argument('--oauth-config', type = json.loads, help = 'Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })', required = False) - # TODO test CLI passing of args - - parser_render = subcommands.add_parser('render', description="Renders a script to the console, used to check and verify jinja output from scripts.") - parser_render.add_argument('--config-folder', type = str, default = '.', help = 'The folder to look in for the schemachange-config.yml file (the default is the current working directory)', required = False) - parser_render.add_argument('-f', '--root-folder', type = str, help = 'The root folder for the database change scripts', required = False) - parser_render.add_argument('-m', '--modules-folder', type = str, help = 'The modules folder for jinja macros and templates to be used across multiple scripts', required = False) - parser_render.add_argument('--vars', type = json.loads, help = 'Define values for the variables to replaced in change scripts, given in JSON format (e.g. {"variable1": "value1", "variable2": "value2"})', required = False) - parser_render.add_argument('-v', '--verbose', action='store_true', help = 'Display verbose debugging details during execution (the default is False)', required = False) - parser_render.add_argument('script', type = str, help = 'The script to render') - - # The original parameters did not support subcommands. Check if a subcommand has been supplied - # if not default to deploy to match original behaviour. - args = argv[1:] - if len(args) == 0 or not any(subcommand in args[0].upper() for subcommand in ["DEPLOY", "RENDER"]): - args = ["deploy"] + args - - args = parser.parse_args(args) - - print("schemachange version: %s" % _schemachange_version) - - # First get the config values - config_file_path = os.path.join(args.config_folder, _config_file_name) - - # Retreive argparser attributes as dictionary - schemachange_args = args.__dict__ - schemachange_args['config_file_path'] = config_file_path - - #nullify expected null values for render. - if args.subcommand == 'render': - renderoveride = {"snowflake_account":None,"snowflake_user":None,"snowflake_role":None, \ - "snowflake_warehouse":None,"snowflake_database":None,"change_history_table":None, \ - "snowflake_schema":None,"create_change_history_table":None,"autocommit":None, \ - "dry_run":None,"query_tag":None,"oauth_config":None } - schemachange_args.update(renderoveride) - config = get_schemachange_config(**schemachange_args) - - # setup a secret manager and assign to global scope - sm = SecretManager() - SecretManager.set_global_manager(sm) - # Extract all secrets for --vars - sm.add_range(extract_config_secrets(config)) - - # Then log some details - print("Using root folder %s" % config['root_folder']) - if config['modules_folder']: - print("Using Jinja modules folder %s" % config['modules_folder']) - - # pretty print the variables in yaml style - if config['vars'] == {}: - print("Using variables: {}") - else: - print("Using variables:") - print(textwrap.indent( \ - SecretManager.global_redact(yaml.dump( \ - config['vars'], \ - sort_keys=False, \ - default_flow_style=False)), prefix = " ")) - - # Finally, execute the command - if args.subcommand == 'render': - render_command(config, args.script) - else: - deploy_command(config) + parser = argparse.ArgumentParser( + prog="schemachange", + description="Apply schema changes to a Snowflake account. Full readme at https://github.com/Snowflake-Labs/schemachange", + formatter_class=argparse.RawTextHelpFormatter, + ) + subcommands = parser.add_subparsers(dest="subcommand") + + parser_deploy = subcommands.add_parser("deploy") + parser_deploy.add_argument( + "--config-folder", + type=str, + default=".", + help="The folder to look in for the schemachange-config.yml file (the default is the current working directory)", + required=False, + ) + parser_deploy.add_argument( + "-f", + "--root-folder", + type=str, + help="The root folder for the database change scripts", + required=False, + ) + parser_deploy.add_argument( + "-m", + "--modules-folder", + type=str, + help="The modules folder for jinja macros and templates to be used across multiple scripts", + required=False, + ) + parser_deploy.add_argument( + "-a", + "--snowflake-account", + type=str, + help="The name of the snowflake account (e.g. xy12345.east-us-2.azure)", + required=False, + ) + parser_deploy.add_argument( + "-u", + "--snowflake-user", + type=str, + help="The name of the snowflake user", + required=False, + ) + parser_deploy.add_argument( + "-r", + "--snowflake-role", + type=str, + help="The name of the default role to use", + required=False, + ) + parser_deploy.add_argument( + "-w", + "--snowflake-warehouse", + type=str, + help="The name of the default warehouse to use. Can be overridden in the change scripts.", + required=False, + ) + parser_deploy.add_argument( + "-d", + "--snowflake-database", + type=str, + help="The name of the default database to use. Can be overridden in the change scripts.", + required=False, + ) + parser_deploy.add_argument( + "-s", + "--snowflake-schema", + type=str, + help="The name of the default schema to use. Can be overridden in the change scripts.", + required=False, + ) + parser_deploy.add_argument( + "-c", + "--change-history-table", + type=str, + help="Used to override the default name of the change history table (the default is METADATA.SCHEMACHANGE.CHANGE_HISTORY)", + required=False, + ) + parser_deploy.add_argument( + "--vars", + type=json.loads, + help='Define values for the variables to replaced in change scripts, given in JSON format (e.g. {"variable1": "value1", "variable2": "value2"})', + required=False, + ) + parser_deploy.add_argument( + "--create-change-history-table", + action="store_true", + help="Create the change history schema and table, if they do not exist (the default is False)", + required=False, + ) + parser_deploy.add_argument( + "-ac", + "--autocommit", + action="store_true", + help="Enable autocommit feature for DML commands (the default is False)", + required=False, + ) + parser_deploy.add_argument( + "-v", + "--verbose", + action="store_true", + help="Display verbose debugging details during execution (the default is False)", + required=False, + ) + parser_deploy.add_argument( + "--dry-run", + action="store_true", + help="Run schemachange in dry run mode (the default is False)", + required=False, + ) + parser_deploy.add_argument( + "--query-tag", + type=str, + help="The string to add to the Snowflake QUERY_TAG session value for each query executed", + required=False, + ) + parser_deploy.add_argument( + "--oauth-config", + type=json.loads, + help='Define values for the variables to Make Oauth Token requests (e.g. {"token-provider-url": "https//...", "token-request-payload": {"client_id": "GUID_xyz",...},... })', + required=False, + ) + # TODO test CLI passing of args + + parser_render = subcommands.add_parser( + "render", + description="Renders a script to the console, used to check and verify jinja output from scripts.", + ) + parser_render.add_argument( + "--config-folder", + type=str, + default=".", + help="The folder to look in for the schemachange-config.yml file (the default is the current working directory)", + required=False, + ) + parser_render.add_argument( + "-f", + "--root-folder", + type=str, + help="The root folder for the database change scripts", + required=False, + ) + parser_render.add_argument( + "-m", + "--modules-folder", + type=str, + help="The modules folder for jinja macros and templates to be used across multiple scripts", + required=False, + ) + parser_render.add_argument( + "--vars", + type=json.loads, + help='Define values for the variables to replaced in change scripts, given in JSON format (e.g. {"variable1": "value1", "variable2": "value2"})', + required=False, + ) + parser_render.add_argument( + "-v", + "--verbose", + action="store_true", + help="Display verbose debugging details during execution (the default is False)", + required=False, + ) + parser_render.add_argument("script", type=str, help="The script to render") + + # The original parameters did not support subcommands. Check if a subcommand has been supplied + # if not default to deploy to match original behaviour. + args = argv[1:] + if len(args) == 0 or not any( + subcommand in args[0].upper() for subcommand in ["DEPLOY", "RENDER"] + ): + args = ["deploy"] + args + + args = parser.parse_args(args) + + print("schemachange version: %s" % _schemachange_version) + + # First get the config values + config_file_path = os.path.join(args.config_folder, _config_file_name) + + # Retreive argparser attributes as dictionary + schemachange_args = args.__dict__ + schemachange_args["config_file_path"] = config_file_path + + # nullify expected null values for render. + if args.subcommand == "render": + renderoveride = { + "snowflake_account": None, + "snowflake_user": None, + "snowflake_role": None, + "snowflake_warehouse": None, + "snowflake_database": None, + "change_history_table": None, + "snowflake_schema": None, + "create_change_history_table": None, + "autocommit": None, + "dry_run": None, + "query_tag": None, + "oauth_config": None, + } + schemachange_args.update(renderoveride) + config = get_schemachange_config(**schemachange_args) + + # setup a secret manager and assign to global scope + sm = SecretManager() + SecretManager.set_global_manager(sm) + # Extract all secrets for --vars + sm.add_range(extract_config_secrets(config)) + + # Then log some details + print("Using root folder %s" % config["root_folder"]) + if config["modules_folder"]: + print("Using Jinja modules folder %s" % config["modules_folder"]) + + # pretty print the variables in yaml style + if config["vars"] == {}: + print("Using variables: {}") + else: + print("Using variables:") + print( + textwrap.indent( + SecretManager.global_redact( + yaml.dump(config["vars"], sort_keys=False, default_flow_style=False) + ), + prefix=" ", + ) + ) + + # Finally, execute the command + if args.subcommand == "render": + render_command(config, args.script) + else: + deploy_command(config) + if __name__ == "__main__": - main() + main() From e413286500e5d4e532b39c1adcb427e91e7947fc Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 25 Mar 2024 11:12:47 -0400 Subject: [PATCH 05/14] updating the cli with formatted with black formatter --- .pre-commit-config.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9639d152..18701f8c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,12 +24,12 @@ repos: # Run the formatter. - id: ruff-format # Using this mirror lets us use mypyc-compiled black, which is about 2x faster -# - repo: https://github.com/psf/black-pre-commit-mirror -# rev: 24.3.0 -# hooks: -# - id: black -# # It is recommended to specify the latest version of Python -# # supported by your project here, or alternatively use -# # pre-commit's default_language_version, see -# # https://pre-commit.com/#top_level-default_language_version -# language_version: python3.10 \ No newline at end of file +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.3.0 + hooks: + - id: black + # It is recommended to specify the latest version of Python + # supported by your project here, or alternatively use + # pre-commit's default_language_version, see + # https://pre-commit.com/#top_level-default_language_version + language_version: python3.10 \ No newline at end of file From e063ce0f7c5f2cc65b3bf365230976aaa324ddd4 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 25 Mar 2024 11:32:07 -0400 Subject: [PATCH 06/14] push black formatter as well --- .pre-commit-config.yaml | 52 ++++++++++++++++++++--------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 18701f8c..2ff9c8c4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,35 +1,35 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: -- repo: https://github.com/pre-commit/pre-commit-hooks + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.2.0 hooks: - - id: trailing-whitespace - - id: end-of-file-fixer -- repo: https://github.com/asottile/setup-cfg-fmt + - id: trailing-whitespace + - id: end-of-file-fixer + - repo: https://github.com/asottile/setup-cfg-fmt rev: v2.4.0 hooks: - - id: setup-cfg-fmt -- repo: https://github.com/asottile/pyupgrade + - id: setup-cfg-fmt + - repo: https://github.com/asottile/pyupgrade rev: v3.15.2 hooks: - - id: pyupgrade -- repo: https://github.com/astral-sh/ruff-pre-commit - # Ruff version. - rev: v0.3.4 - hooks: - # Run the linter. - - id: ruff - args: [ --fix ] - # Run the formatter. - - id: ruff-format -# Using this mirror lets us use mypyc-compiled black, which is about 2x faster -- repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.3.0 - hooks: - - id: black - # It is recommended to specify the latest version of Python - # supported by your project here, or alternatively use - # pre-commit's default_language_version, see - # https://pre-commit.com/#top_level-default_language_version - language_version: python3.10 \ No newline at end of file + - id: pyupgrade + - repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.3.4 + hooks: + # Run the linter. + - id: ruff + args: [ --fix ] + # Run the formatter. + - id: ruff-format + # Using this mirror lets us use mypyc-compiled black, which is about 2x faster + - repo: https://github.com/psf/black-pre-commit-mirror + rev: 24.3.0 + hooks: + - id: black + # It is recommended to specify the latest version of Python + # supported by your project here, or alternatively use + # pre-commit's default_language_version, see + # https://pre-commit.com/#top_level-default_language_version + language_version: python3.11 From 0274ac6fc5405997a0ef5f1614684c70a374f74b Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 25 Mar 2024 12:05:02 -0400 Subject: [PATCH 07/14] moved snowflake params call to its own method. --- schemachange/cli.py | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/schemachange/cli.py b/schemachange/cli.py index f321b789..ca36feaa 100644 --- a/schemachange/cli.py +++ b/schemachange/cli.py @@ -274,21 +274,8 @@ class SnowflakeSchemachangeSession: # endregion Query Templates def __init__(self, config): - session_parameters = {"QUERY_TAG": "schemachange %s" % _schemachange_version} - if config["query_tag"]: - session_parameters["QUERY_TAG"] += ";%s" % config["query_tag"] - # Retreive Connection info from config dictionary - self.conArgs = { - "user": config["snowflake_user"], - "account": config["snowflake_account"], - "role": config["snowflake_role"], - "warehouse": config["snowflake_warehouse"], - "database": config["snowflake_database"], - "schema": config["snowflake_schema"], - "application": _snowflake_application_name, - "session_parameters": session_parameters, - } + self.conArgs = self.get_snowflake_params(config) self.oauth_config = config["oauth_config"] self.autocommit = config["autocommit"] @@ -304,6 +291,23 @@ def __del__(self): if hasattr(self, "con"): self.con.close() + def get_snowflake_params(self, config): + + session_parameters = {"QUERY_TAG": "schemachange %s" % _schemachange_version} + if config["query_tag"]: + session_parameters["QUERY_TAG"] += ";%s" % config["query_tag"] + + return { + "user": config["snowflake_user"], + "account": config["snowflake_account"], + "role": config["snowflake_role"], + "warehouse": config["snowflake_warehouse"], + "database": config["snowflake_database"], + "schema": config["snowflake_schema"], + "application": _snowflake_application_name, + "session_parameters": session_parameters, + } + def get_oauth_token(self): req_info = { "url": self.oauth_config["token-provider-url"], From 7523150fe9eb3db29cb6a9d786afe1cc0a5c4e4e Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 25 Mar 2024 12:44:56 -0400 Subject: [PATCH 08/14] formatted tests with black formatter. --- tests/test_JinjaEnvVar.py | 23 ++- tests/test_JinjaTemplateProcessor.py | 4 +- tests/test_SecretManager.py | 3 + tests/test_cli_misc.py | 44 ++++ tests/test_extract_config_secrets.py | 56 +++-- tests/test_get_all_scripts_recursively.py | 8 +- tests/test_jinja_env_var_template.py | 5 +- tests/test_load_schemachange_config.py | 28 ++- tests/test_main.py | 236 ++++++++++++++-------- 9 files changed, 287 insertions(+), 120 deletions(-) create mode 100644 tests/test_cli_misc.py diff --git a/tests/test_JinjaEnvVar.py b/tests/test_JinjaEnvVar.py index d67777a5..dbaaf94e 100644 --- a/tests/test_JinjaEnvVar.py +++ b/tests/test_JinjaEnvVar.py @@ -9,32 +9,37 @@ @mock.patch.dict(os.environ, {}, clear=True) def test_env_var_with_no_default_and_no_environmental_variables_should_raise_exception(): - assert ('SF_DATABASE' in os.environ) is False + assert ("SF_DATABASE" in os.environ) is False with pytest.raises(ValueError) as e: - result = JinjaEnvVar.env_var('SF_DATABASE') - assert str(e.value) == "Could not find environmental variable SF_DATABASE and no default value was provided" + result = JinjaEnvVar.env_var("SF_DATABASE") + assert ( + str(e.value) + == "Could not find environmental variable SF_DATABASE and no default value was provided" + ) @mock.patch.dict(os.environ, {}, clear=True) def test_env_var_with_default_and_no_environmental_variables_should_return_default(): print(os.environ) - assert ('SF_DATABASE' in os.environ) is False + assert ("SF_DATABASE" in os.environ) is False - result = JinjaEnvVar.env_var('SF_DATABASE', 'SCHEMACHANGE_DEMO') - assert result == 'SCHEMACHANGE_DEMO' + result = JinjaEnvVar.env_var("SF_DATABASE", "SCHEMACHANGE_DEMO") + assert result == "SCHEMACHANGE_DEMO" @mock.patch.dict(os.environ, {"SF_DATABASE": "SCHEMACHANGE_DEMO_2"}, clear=True) def test_env_var_with_default_and_environmental_variables_should_return_environmental_variable_value(): - result = JinjaEnvVar.env_var('SF_DATABASE', 'SCHEMACHANGE_DEMO') - assert result == 'SCHEMACHANGE_DEMO_2' + result = JinjaEnvVar.env_var("SF_DATABASE", "SCHEMACHANGE_DEMO") + assert result == "SCHEMACHANGE_DEMO_2" @mock.patch.dict(os.environ, {"SF_DATABASE": "SCHEMACHANGE_DEMO_3"}, clear=True) def test_JinjaEnvVar_with_jinja_template(): - template = jinja2.Template("{{env_var('SF_DATABASE', 'SCHEMACHANGE_DEMO')}}", extensions=[JinjaEnvVar]) + template = jinja2.Template( + "{{env_var('SF_DATABASE', 'SCHEMACHANGE_DEMO')}}", extensions=[JinjaEnvVar] + ) assert template.render() == "SCHEMACHANGE_DEMO_3" diff --git a/tests/test_JinjaTemplateProcessor.py b/tests/test_JinjaTemplateProcessor.py index f6bcb1c5..1b1e5ab6 100644 --- a/tests/test_JinjaTemplateProcessor.py +++ b/tests/test_JinjaTemplateProcessor.py @@ -2,7 +2,7 @@ import pathlib import pytest -from jinja2 import DictLoader +from jinja2 import DictLoader from jinja2.exceptions import UndefinedError from schemachange.cli import JinjaTemplateProcessor @@ -51,7 +51,7 @@ def test_JinjaTemplateProcessor_render_from_subfolder(tmp_path: pathlib.Path): root_folder = tmp_path / "MORE2" root_folder.mkdir() - script_folder = root_folder/ "SQL" + script_folder = root_folder / "SQL" script_folder.mkdir() script_file = script_folder / "1.0.0_my_test.sql" script_file.write_text("Hello world!") diff --git a/tests/test_SecretManager.py b/tests/test_SecretManager.py index 2ed70acb..2df8326d 100644 --- a/tests/test_SecretManager.py +++ b/tests/test_SecretManager.py @@ -43,6 +43,7 @@ def test_SecretManager_given_one_secrets_when_add_range_with_None_then_Count_sho assert len(sm._SecretManager__secrets) == 1 + def test_SecretManager_given_one_secrets_when_add_range_with_empty_set_then_Count_should_remain_one(): sm = SecretManager() sm.add("world") @@ -52,6 +53,7 @@ def test_SecretManager_given_one_secrets_when_add_range_with_empty_set_then_Coun assert len(sm._SecretManager__secrets) == 1 + def test_SecretManager_given_one_secrets_when_add_range_with_two_secrets_then_count_of_secrets_three(): sm = SecretManager() sm.add("world") @@ -68,6 +70,7 @@ def test_SecretManager_given_one_secrets_when_add_range_with_two_secrets_then_co ##### test static methods ##### + def test_SecretManager_check_global_assignment_round_trip(): sm = SecretManager() diff --git a/tests/test_cli_misc.py b/tests/test_cli_misc.py new file mode 100644 index 00000000..f0801f65 --- /dev/null +++ b/tests/test_cli_misc.py @@ -0,0 +1,44 @@ +import pytest +import schemachange.cli + + +def test_alphanum_convert_integer(): + assert schemachange.cli.alphanum_convert("123") == 123 + + +def test_alphanum_convert_lowercase(): + assert schemachange.cli.alphanum_convert("TEST") == "test" + + +def test_get_alphanum_key_empty_string(): + assert schemachange.cli.get_alphanum_key("") == [""] + + +def test_get_alphanum_key_numbers_only(): + assert schemachange.cli.get_alphanum_key("123") == ["", 123, ""] + + +def test_get_alphanum_key_alphabets_only(): + assert schemachange.cli.get_alphanum_key("abc") == ["abc"] + + +def test_get_alphanum_key_upper_alphanumeric(): + assert schemachange.cli.get_alphanum_key("V1.2.3__") == [ + "v", + 1, + ".", + 2, + ".", + 3, + "__", + ] + + +def test_get_alphanum_key_valid_version_string(): + assert schemachange.cli.get_alphanum_key("1.2.2") == ["", 1, ".", 2, ".", 2, ""] + + +def test_sorted_alphanumeric_mixed_string(): + assert schemachange.cli.sorted_alphanumeric( + ["V1.2.3__file.sql", "V1.2.4__file.sql"] + ) == ["V1.2.3__file.sql", "V1.2.4__file.sql"] diff --git a/tests/test_extract_config_secrets.py b/tests/test_extract_config_secrets.py index 91560746..94521705 100644 --- a/tests/test_extract_config_secrets.py +++ b/tests/test_extract_config_secrets.py @@ -1,6 +1,7 @@ import pytest from schemachange.cli import extract_config_secrets + def test_extract_config_secrets_given_empty_config_should_not_error(): config = {} results = extract_config_secrets(config) @@ -10,21 +11,34 @@ def test_extract_config_secrets_given_None_should_not_error(): results = extract_config_secrets(None) -@pytest.mark.parametrize("config, secret", [ - ({"vars": {"secret" : "secret_val1"}}, "secret_val1"), - ({"vars": {"SECret" : "secret_val2"}}, "secret_val2"), - ({"vars": {"secret_key" : "secret_val3"}}, "secret_val3"), - ({"vars": {"s3_bucket_secret" : "secret_val4"}}, "secret_val4"), - ({"vars": {"s3SecretKey" : "secret_val5"}}, "secret_val5"), - ({"vars": { "nested" : {"s3_bucket_secret" : "secret_val6"}}}, "secret_val6"), -]) -def test_extract_config_secrets_given__vars_with_keys_should_extract_secret(config, secret): +@pytest.mark.parametrize( + "config, secret", + [ + ({"vars": {"secret": "secret_val1"}}, "secret_val1"), + ({"vars": {"SECret": "secret_val2"}}, "secret_val2"), + ({"vars": {"secret_key": "secret_val3"}}, "secret_val3"), + ({"vars": {"s3_bucket_secret": "secret_val4"}}, "secret_val4"), + ({"vars": {"s3SecretKey": "secret_val5"}}, "secret_val5"), + ({"vars": {"nested": {"s3_bucket_secret": "secret_val6"}}}, "secret_val6"), + ], +) +def test_extract_config_secrets_given__vars_with_keys_should_extract_secret( + config, secret +): results = extract_config_secrets(config) assert secret in results def test_extract_config_secrets_given__vars_with_secrets_key_then_all_children_should_be_treated_as_secrets(): - config = {"vars": {"secrets" : {"database_name": "database_name_val", "schema_name": "schema_name_val", "nested_secrets": {"SEC_ONE": "SEC_ONE_VAL" } } }} + config = { + "vars": { + "secrets": { + "database_name": "database_name_val", + "schema_name": "schema_name_val", + "nested_secrets": {"SEC_ONE": "SEC_ONE_VAL"}, + } + } + } results = extract_config_secrets(config) @@ -35,7 +49,17 @@ def test_extract_config_secrets_given__vars_with_secrets_key_then_all_children_s def test_extract_config_secrets_given__vars_with_nested_secrets_key_then_all_children_should_be_treated_as_secrets(): - config = {"vars": {"nested": {"secrets" : {"database_name": "database_name_val", "schema_name": "schema_name_val", "nested": {"SEC_ONE": "SEC_ONE_VAL" } } } }} + config = { + "vars": { + "nested": { + "secrets": { + "database_name": "database_name_val", + "schema_name": "schema_name_val", + "nested": {"SEC_ONE": "SEC_ONE_VAL"}, + } + } + } + } results = extract_config_secrets(config) @@ -46,7 +70,15 @@ def test_extract_config_secrets_given__vars_with_nested_secrets_key_then_all_chi def test_extract_config_secrets_given__vars_with_same_secret_twice_then_only_extracted_once(): - config = {"vars": {"secrets" : {"database_name": "SECRET_VALUE", "schema_name": "SECRET_VALUE", "nested_secrets": {"SEC_ONE": "SECRET_VALUE" } } }} + config = { + "vars": { + "secrets": { + "database_name": "SECRET_VALUE", + "schema_name": "SECRET_VALUE", + "nested_secrets": {"SEC_ONE": "SECRET_VALUE"}, + } + } + } results = extract_config_secrets(config) diff --git a/tests/test_get_all_scripts_recursively.py b/tests/test_get_all_scripts_recursively.py index 2d3baf60..26cd8b63 100644 --- a/tests/test_get_all_scripts_recursively.py +++ b/tests/test_get_all_scripts_recursively.py @@ -103,7 +103,7 @@ def test_get_all_scripts_recursively__given_single_Version_jinja_file_should_ext def test_get_all_scripts_recursively__given_same_version_file_with_and_without_jinja_extension_should_raise_exception(): - with mock.patch("os.walk") as mockwalk: + with mock.patch("os.walk") as mockwalk: mockwalk.return_value = [ ("", (""), ("V1.1.1__intial.sql", "V1.1.1__intial.sql.jinja")), ] @@ -114,6 +114,7 @@ def test_get_all_scripts_recursively__given_same_version_file_with_and_without_j "The script name V1.1.1__intial.sql exists more than once (first_instance" ) + ########################### #### Always file tests #### ########################### @@ -186,7 +187,7 @@ def test_get_all_scripts_recursively__given_single_Always_jinja_file_should_extr def test_get_all_scripts_recursively__given_same_Always_file_with_and_without_jinja_extension_should_raise_exception(): - with mock.patch("os.walk") as mockwalk: + with mock.patch("os.walk") as mockwalk: mockwalk.return_value = [ ("", (""), ("A__intial.sql", "A__intial.sql.jinja")), ] @@ -197,6 +198,7 @@ def test_get_all_scripts_recursively__given_same_Always_file_with_and_without_ji "The script name A__intial.sql exists more than once (first_instance " ) + ############################### #### Repeatable file tests #### ############################### @@ -269,7 +271,7 @@ def test_get_all_scripts_recursively__given_single_Repeatable_jinja_file_should_ def test_get_all_scripts_recursively__given_same_Repeatable_file_with_and_without_jinja_extension_should_raise_exception(): - with mock.patch("os.walk") as mockwalk: + with mock.patch("os.walk") as mockwalk: mockwalk.return_value = [ ("", (""), ("R__intial.sql", "R__intial.sql.jinja")), ] diff --git a/tests/test_jinja_env_var_template.py b/tests/test_jinja_env_var_template.py index 52f26b02..7b7d7b01 100644 --- a/tests/test_jinja_env_var_template.py +++ b/tests/test_jinja_env_var_template.py @@ -16,7 +16,10 @@ def test_from_environ_not_set(): with pytest.raises(ValueError) as e: context = processor.render("test.sql", None, True) - assert str(e.value) == "Could not find environmental variable MYVAR and no default value was provided" + assert ( + str(e.value) + == "Could not find environmental variable MYVAR and no default value was provided" + ) def test_from_environ_set(): diff --git a/tests/test_load_schemachange_config.py b/tests/test_load_schemachange_config.py index 04ad50bd..a030e475 100644 --- a/tests/test_load_schemachange_config.py +++ b/tests/test_load_schemachange_config.py @@ -5,8 +5,10 @@ import pytest from schemachange.cli import load_schemachange_config + # Note Paramters in config file are kebab case and are re-rendered as snake case after 'load_schemachange_config' is called + def test__load_schemachange_config__simple_config_file(tmp_path: pathlib.Path): config_contents = """ @@ -19,17 +21,18 @@ def test__load_schemachange_config__simple_config_file(tmp_path: pathlib.Path): config_file = tmp_path / "schemachange-config.yml" config_file.write_text(config_contents) - config = load_schemachange_config(str(config_file)) - assert config['config-version'] == 1 - assert config['root-folder'] == 'scripts' - assert config['modules-folder'] == 'modules' - assert config['vars']['database_name'] == 'SCHEMACHANGE_DEMO_JINJA' + assert config["config-version"] == 1 + assert config["root-folder"] == "scripts" + assert config["modules-folder"] == "modules" + assert config["vars"]["database_name"] == "SCHEMACHANGE_DEMO_JINJA" @mock.patch.dict(os.environ, {"TEST_VAR": "env_value"}) -def test__load_schemachange_config__with_env_var_should_populate_value(tmp_path: pathlib.Path): +def test__load_schemachange_config__with_env_var_should_populate_value( + tmp_path: pathlib.Path, +): config_contents = """ config-version: 1.1 @@ -43,10 +46,12 @@ def test__load_schemachange_config__with_env_var_should_populate_value(tmp_path: config = load_schemachange_config(str(config_file)) - assert config['root-folder'] == 'env_value' + assert config["root-folder"] == "env_value" -def test__load_schemachange_config__requiring_env_var_but_env_var_not_set_should_raise_exception(tmp_path: pathlib.Path): +def test__load_schemachange_config__requiring_env_var_but_env_var_not_set_should_raise_exception( + tmp_path: pathlib.Path, +): config_contents = """ config-version: 1.1 @@ -59,5 +64,8 @@ def test__load_schemachange_config__requiring_env_var_but_env_var_not_set_should config_file.write_text(config_contents) with pytest.raises(ValueError) as e: - config = load_schemachange_config(str(config_file)) - assert str(e.value) == "Could not find environmental variable TEST_VAR and no default value was provided" + config = load_schemachange_config(str(config_file)) + assert ( + str(e.value) + == "Could not find environmental variable TEST_VAR and no default value was provided" + ) diff --git a/tests/test_main.py b/tests/test_main.py index 39217fe3..b55a7b61 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -8,79 +8,131 @@ import schemachange.cli DEFAULT_CONFIG = { - 'root_folder': os.path.abspath('.'), - 'modules_folder': None, - 'snowflake_account': None, - 'snowflake_user': None, - 'snowflake_role':None, - 'snowflake_warehouse': None, - 'snowflake_database': None, - 'snowflake_schema': None, - 'change_history_table': None, - 'vars': {}, - 'create_change_history_table': False, - 'autocommit': False, - 'verbose': False, - 'dry_run': False, - 'query_tag': None, - 'oauth_config':None, + "root_folder": os.path.abspath("."), + "modules_folder": None, + "snowflake_account": None, + "snowflake_user": None, + "snowflake_role": None, + "snowflake_warehouse": None, + "snowflake_database": None, + "snowflake_schema": None, + "change_history_table": None, + "vars": {}, + "create_change_history_table": False, + "autocommit": False, + "verbose": False, + "dry_run": False, + "query_tag": None, + "oauth_config": None, } -@pytest.mark.parametrize("args, expected", [ - (["schemachange"], DEFAULT_CONFIG), - (["schemachange", "deploy"], DEFAULT_CONFIG), - (["schemachange", "deploy", "-f", '.'], - {**DEFAULT_CONFIG, 'root_folder':os.path.abspath('.')}), - (["schemachange", "deploy", "--snowflake-account", "account"], - {**DEFAULT_CONFIG, 'snowflake_account': 'account'}), - (["schemachange", "deploy", "--snowflake-user", "user"], - {**DEFAULT_CONFIG, 'snowflake_user': 'user'}), - (["schemachange", "deploy", "--snowflake-role", "role"], - {**DEFAULT_CONFIG, 'snowflake_role': 'role'}), - (["schemachange", "deploy", "--snowflake-warehouse", "warehouse"], - {**DEFAULT_CONFIG, 'snowflake_warehouse': 'warehouse'}), - (["schemachange", "deploy", "--snowflake-database", "database"], - {**DEFAULT_CONFIG, 'snowflake_database': 'database'}), - (["schemachange", "deploy", "--snowflake-schema", "schema"], - {**DEFAULT_CONFIG, "snowflake_schema": "schema"}), - (["schemachange", "deploy", "--change-history-table", "db.schema.table"], - {**DEFAULT_CONFIG, 'change_history_table': 'db.schema.table'}), - (["schemachange", "deploy", "--vars", '{"var1": "val"}'], - {**DEFAULT_CONFIG, 'vars': {'var1' : 'val'},}), - (["schemachange", "deploy", "--create-change-history-table"], - {**DEFAULT_CONFIG, 'create_change_history_table': True}), - (["schemachange", "deploy", "--autocommit"], - {**DEFAULT_CONFIG, 'autocommit': True}), - (["schemachange", "deploy", "--verbose"], - {**DEFAULT_CONFIG, 'verbose': True}), - (["schemachange", "deploy", "--dry-run"], - {**DEFAULT_CONFIG, 'dry_run': True}), - (["schemachange", "deploy", "--query-tag", "querytag"], - {**DEFAULT_CONFIG, 'query_tag': 'querytag'}), - (["schemachange", "deploy", "--oauth-config", '{"token-provider-url": "https//..."}'], - {**DEFAULT_CONFIG, 'oauth_config': {"token-provider-url": "https//..."},}), -]) -def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): +@pytest.mark.parametrize( + "args, expected", + [ + (["schemachange"], DEFAULT_CONFIG), + (["schemachange", "deploy"], DEFAULT_CONFIG), + ( + ["schemachange", "deploy", "-f", "."], + {**DEFAULT_CONFIG, "root_folder": os.path.abspath(".")}, + ), + ( + ["schemachange", "deploy", "--snowflake-account", "account"], + {**DEFAULT_CONFIG, "snowflake_account": "account"}, + ), + ( + ["schemachange", "deploy", "--snowflake-user", "user"], + {**DEFAULT_CONFIG, "snowflake_user": "user"}, + ), + ( + ["schemachange", "deploy", "--snowflake-role", "role"], + {**DEFAULT_CONFIG, "snowflake_role": "role"}, + ), + ( + ["schemachange", "deploy", "--snowflake-warehouse", "warehouse"], + {**DEFAULT_CONFIG, "snowflake_warehouse": "warehouse"}, + ), + ( + ["schemachange", "deploy", "--snowflake-database", "database"], + {**DEFAULT_CONFIG, "snowflake_database": "database"}, + ), + ( + ["schemachange", "deploy", "--snowflake-schema", "schema"], + {**DEFAULT_CONFIG, "snowflake_schema": "schema"}, + ), + ( + ["schemachange", "deploy", "--change-history-table", "db.schema.table"], + {**DEFAULT_CONFIG, "change_history_table": "db.schema.table"}, + ), + ( + ["schemachange", "deploy", "--vars", '{"var1": "val"}'], + { + **DEFAULT_CONFIG, + "vars": {"var1": "val"}, + }, + ), + ( + ["schemachange", "deploy", "--create-change-history-table"], + {**DEFAULT_CONFIG, "create_change_history_table": True}, + ), + ( + ["schemachange", "deploy", "--autocommit"], + {**DEFAULT_CONFIG, "autocommit": True}, + ), + (["schemachange", "deploy", "--verbose"], {**DEFAULT_CONFIG, "verbose": True}), + (["schemachange", "deploy", "--dry-run"], {**DEFAULT_CONFIG, "dry_run": True}), + ( + ["schemachange", "deploy", "--query-tag", "querytag"], + {**DEFAULT_CONFIG, "query_tag": "querytag"}, + ), + ( + [ + "schemachange", + "deploy", + "--oauth-config", + '{"token-provider-url": "https//..."}', + ], + { + **DEFAULT_CONFIG, + "oauth_config": {"token-provider-url": "https//..."}, + }, + ), + ], +) +def test_main_deploy_subcommand_given_arguments_make_sure_arguments_set_on_call( + args, expected +): with mock.patch("schemachange.cli.deploy_command") as mock_deploy_command: schemachange.cli.main(args) mock_deploy_command.assert_called_once() - [config,], _call_kwargs = mock_deploy_command.call_args + [ + config, + ], _call_kwargs = mock_deploy_command.call_args assert config == expected -@pytest.mark.parametrize("args, expected", [ - (["schemachange", "render", "script.sql"], - ({**DEFAULT_CONFIG}, "script.sql")), - (["schemachange", "render", "--root-folder", '.', "script.sql"], - ({**DEFAULT_CONFIG, 'root_folder': os.path.abspath('.')}, "script.sql")), - (["schemachange", "render", "--vars", '{"var1": "val"}', "script.sql"], - ({**DEFAULT_CONFIG, 'vars': {"var1": "val"}}, "script.sql")), - (["schemachange", "render", "--verbose", "script.sql"], - ({**DEFAULT_CONFIG, 'verbose': True}, "script.sql")), -]) -def test_main_render_subcommand_given_arguments_make_sure_arguments_set_on_call( args, expected): +@pytest.mark.parametrize( + "args, expected", + [ + (["schemachange", "render", "script.sql"], ({**DEFAULT_CONFIG}, "script.sql")), + ( + ["schemachange", "render", "--root-folder", ".", "script.sql"], + ({**DEFAULT_CONFIG, "root_folder": os.path.abspath(".")}, "script.sql"), + ), + ( + ["schemachange", "render", "--vars", '{"var1": "val"}', "script.sql"], + ({**DEFAULT_CONFIG, "vars": {"var1": "val"}}, "script.sql"), + ), + ( + ["schemachange", "render", "--verbose", "script.sql"], + ({**DEFAULT_CONFIG, "verbose": True}, "script.sql"), + ), + ], +) +def test_main_render_subcommand_given_arguments_make_sure_arguments_set_on_call( + args, expected +): with mock.patch("schemachange.cli.render_command") as mock_render_command: schemachange.cli.main(args) @@ -89,20 +141,31 @@ def test_main_render_subcommand_given_arguments_make_sure_arguments_set_on_call( assert call_args == expected -@pytest.mark.parametrize("args, to_mock, expected_args", [ - (["schemachange", "deploy", "--config-folder", "DUMMY"], - "schemachange.cli.deploy_command", - ({**DEFAULT_CONFIG, 'snowflake_account': 'account'},)), - (["schemachange", "render", "script.sql", "--config-folder", "DUMMY"], - "schemachange.cli.render_command", - ({**DEFAULT_CONFIG, 'snowflake_account': 'account'}, "script.sql")) -]) +@pytest.mark.parametrize( + "args, to_mock, expected_args", + [ + ( + ["schemachange", "deploy", "--config-folder", "DUMMY"], + "schemachange.cli.deploy_command", + ({**DEFAULT_CONFIG, "snowflake_account": "account"},), + ), + ( + ["schemachange", "render", "script.sql", "--config-folder", "DUMMY"], + "schemachange.cli.render_command", + ({**DEFAULT_CONFIG, "snowflake_account": "account"}, "script.sql"), + ), + ], +) def test_main_deploy_config_folder(args, to_mock, expected_args): with tempfile.TemporaryDirectory() as d: - with open(os.path.join(d, 'schemachange-config.yml'), 'wt') as f: - f.write(dedent(''' + with open(os.path.join(d, "schemachange-config.yml"), "wt") as f: + f.write( + dedent( + """ snowflake_account: account - ''')) + """ + ) + ) args[args.index("DUMMY")] = d @@ -113,19 +176,26 @@ def test_main_deploy_config_folder(args, to_mock, expected_args): assert call_args == expected_args -@pytest.mark.parametrize("args, to_mock, expected_args", [ - (["schemachange", "deploy", "--modules-folder", "DUMMY"], - "schemachange.cli.deploy_command", - ({**DEFAULT_CONFIG, 'modules_folder': 'DUMMY'},)), - (["schemachange", "render", "script.sql", "--modules-folder", "DUMMY"], - "schemachange.cli.render_command", - ({**DEFAULT_CONFIG, 'modules_folder': 'DUMMY'}, "script.sql")) -]) +@pytest.mark.parametrize( + "args, to_mock, expected_args", + [ + ( + ["schemachange", "deploy", "--modules-folder", "DUMMY"], + "schemachange.cli.deploy_command", + ({**DEFAULT_CONFIG, "modules_folder": "DUMMY"},), + ), + ( + ["schemachange", "render", "script.sql", "--modules-folder", "DUMMY"], + "schemachange.cli.render_command", + ({**DEFAULT_CONFIG, "modules_folder": "DUMMY"}, "script.sql"), + ), + ], +) def test_main_deploy_modules_folder(args, to_mock, expected_args): with tempfile.TemporaryDirectory() as d: args[args.index("DUMMY")] = d - expected_args[0]['modules_folder'] = d + expected_args[0]["modules_folder"] = d with mock.patch(to_mock) as mock_command: schemachange.cli.main(args) From 54f178d8cb67478aeee124e0c122e58ec39f5c42 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Mon, 25 Mar 2024 12:48:55 -0400 Subject: [PATCH 09/14] Updated black formatter for tests as well. --- tests/test_cli_misc.py | 1 - tests/test_jinja_env_var_template.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/test_cli_misc.py b/tests/test_cli_misc.py index f0801f65..cddfaf07 100644 --- a/tests/test_cli_misc.py +++ b/tests/test_cli_misc.py @@ -1,4 +1,3 @@ -import pytest import schemachange.cli diff --git a/tests/test_jinja_env_var_template.py b/tests/test_jinja_env_var_template.py index 7b7d7b01..e4142f45 100644 --- a/tests/test_jinja_env_var_template.py +++ b/tests/test_jinja_env_var_template.py @@ -1,4 +1,3 @@ -import json import os import pytest From 5d5b8557691f8c3cf8687687cd7942e05bb5a9d0 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Tue, 26 Mar 2024 11:36:00 -0400 Subject: [PATCH 10/14] Updated Test cases Plus Ruff formatted --- tests/test_JinjaEnvVar.py | 2 +- tests/test_JinjaTemplateProcessor.py | 2 +- tests/test_extract_config_secrets.py | 8 ++++---- tests/test_get_all_scripts_recursively.py | 12 ++++++------ tests/test_jinja_env_var_template.py | 2 +- tests/test_load_schemachange_config.py | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test_JinjaEnvVar.py b/tests/test_JinjaEnvVar.py index dbaaf94e..6f17505a 100644 --- a/tests/test_JinjaEnvVar.py +++ b/tests/test_JinjaEnvVar.py @@ -12,7 +12,7 @@ def test_env_var_with_no_default_and_no_environmental_variables_should_raise_exc assert ("SF_DATABASE" in os.environ) is False with pytest.raises(ValueError) as e: - result = JinjaEnvVar.env_var("SF_DATABASE") + JinjaEnvVar.env_var("SF_DATABASE") assert ( str(e.value) == "Could not find environmental variable SF_DATABASE and no default value was provided" diff --git a/tests/test_JinjaTemplateProcessor.py b/tests/test_JinjaTemplateProcessor.py index 1b1e5ab6..072f89e2 100644 --- a/tests/test_JinjaTemplateProcessor.py +++ b/tests/test_JinjaTemplateProcessor.py @@ -27,7 +27,7 @@ def test_JinjaTemplateProcessor_render_simple_string_expecting_variable_that_doe processor.override_loader(DictLoader(templates)) with pytest.raises(UndefinedError) as e: - context = processor.render("test.sql", None, True) + processor.render("test.sql", None, True) assert str(e.value) == "'myvar' is undefined" diff --git a/tests/test_extract_config_secrets.py b/tests/test_extract_config_secrets.py index 94521705..48571029 100644 --- a/tests/test_extract_config_secrets.py +++ b/tests/test_extract_config_secrets.py @@ -2,13 +2,13 @@ from schemachange.cli import extract_config_secrets -def test_extract_config_secrets_given_empty_config_should_not_error(): +def test_extract_config_secrets_given_empty_config_should_produce_empty_set(): config = {} - results = extract_config_secrets(config) + assert len(extract_config_secrets(config)) == 0 -def test_extract_config_secrets_given_None_should_not_error(): - results = extract_config_secrets(None) +def test_extract_config_secrets_given_None_should_produce_empty_set(): + assert len(extract_config_secrets(None)) == 0 @pytest.mark.parametrize( diff --git a/tests/test_get_all_scripts_recursively.py b/tests/test_get_all_scripts_recursively.py index 26cd8b63..4bda33ce 100644 --- a/tests/test_get_all_scripts_recursively.py +++ b/tests/test_get_all_scripts_recursively.py @@ -60,7 +60,7 @@ def test_get_all_scripts_recursively__given_same_Version_twice_should_raise_exce ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + get_all_scripts_recursively("scripts", False) assert str(e.value).startswith( "The script version 1.1.1 exists more than once (second instance" ) @@ -109,7 +109,7 @@ def test_get_all_scripts_recursively__given_same_version_file_with_and_without_j ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + get_all_scripts_recursively("scripts", False) assert str(e.value).startswith( "The script name V1.1.1__intial.sql exists more than once (first_instance" ) @@ -144,7 +144,7 @@ def test_get_all_scripts_recursively__given_same_Always_file_should_raise_except ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + get_all_scripts_recursively("scripts", False) assert str(e.value).startswith( "The script name A__intial.sql exists more than once (first_instance " ) @@ -193,7 +193,7 @@ def test_get_all_scripts_recursively__given_same_Always_file_with_and_without_ji ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + get_all_scripts_recursively("scripts", False) assert str(e.value).startswith( "The script name A__intial.sql exists more than once (first_instance " ) @@ -228,7 +228,7 @@ def test_get_all_scripts_recursively__given_same_Repeatable_file_should_raise_ex ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + get_all_scripts_recursively("scripts", False) assert str(e.value).startswith( "The script name R__intial.sql exists more than once (first_instance " ) @@ -277,7 +277,7 @@ def test_get_all_scripts_recursively__given_same_Repeatable_file_with_and_withou ] with pytest.raises(ValueError) as e: - result = get_all_scripts_recursively("scripts", False) + get_all_scripts_recursively("scripts", False) assert str(e.value).startswith( "The script name R__intial.sql exists more than once (first_instance " ) diff --git a/tests/test_jinja_env_var_template.py b/tests/test_jinja_env_var_template.py index e4142f45..b5a37c4e 100644 --- a/tests/test_jinja_env_var_template.py +++ b/tests/test_jinja_env_var_template.py @@ -13,7 +13,7 @@ def test_from_environ_not_set(): processor.override_loader(DictLoader(templates)) with pytest.raises(ValueError) as e: - context = processor.render("test.sql", None, True) + processor.render("test.sql", None, True) assert ( str(e.value) diff --git a/tests/test_load_schemachange_config.py b/tests/test_load_schemachange_config.py index a030e475..8064d56c 100644 --- a/tests/test_load_schemachange_config.py +++ b/tests/test_load_schemachange_config.py @@ -64,7 +64,7 @@ def test__load_schemachange_config__requiring_env_var_but_env_var_not_set_should config_file.write_text(config_contents) with pytest.raises(ValueError) as e: - config = load_schemachange_config(str(config_file)) + load_schemachange_config(str(config_file)) assert ( str(e.value) == "Could not find environmental variable TEST_VAR and no default value was provided" From d158656716b2323e4becd2debb45cdc75ccb4342 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Tue, 26 Mar 2024 17:24:13 -0400 Subject: [PATCH 11/14] Added ruff cache to ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index cbb22088..1a84845a 100644 --- a/.gitignore +++ b/.gitignore @@ -132,3 +132,5 @@ dmypy.json # Mac files **/.DS_Store + +.ruff_cache/ \ No newline at end of file From 48a3b14e5b633d66d2ce00af41fe6a30972198fc Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 27 Mar 2024 01:43:50 -0400 Subject: [PATCH 12/14] added test cases for change history table. --- tests/test_cli_misc.py | 106 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 99 insertions(+), 7 deletions(-) diff --git a/tests/test_cli_misc.py b/tests/test_cli_misc.py index cddfaf07..b5dd2e75 100644 --- a/tests/test_cli_misc.py +++ b/tests/test_cli_misc.py @@ -1,27 +1,41 @@ import schemachange.cli +import pytest -def test_alphanum_convert_integer(): +def test_cli_given__schemachange_version_change(): + assert schemachange.cli._schemachange_version == "3.6.1" + + +def test_cli_given__constants_exist(): + + assert schemachange.cli._config_file_name == "schemachange-config.yml" + assert schemachange.cli._metadata_database_name == "METADATA" + assert schemachange.cli._metadata_schema_name == "SCHEMACHANGE" + assert schemachange.cli._metadata_table_name == "CHANGE_HISTORY" + assert schemachange.cli._snowflake_application_name == "schemachange" + + +def test_alphanum_convert_given__integer(): assert schemachange.cli.alphanum_convert("123") == 123 -def test_alphanum_convert_lowercase(): +def test_alphanum_convert_given__lowercase(): assert schemachange.cli.alphanum_convert("TEST") == "test" -def test_get_alphanum_key_empty_string(): +def test_get_alphanum_key_given__empty_string(): assert schemachange.cli.get_alphanum_key("") == [""] -def test_get_alphanum_key_numbers_only(): +def test_get_alphanum_key_given__numbers_only(): assert schemachange.cli.get_alphanum_key("123") == ["", 123, ""] -def test_get_alphanum_key_alphabets_only(): +def test_get_alphanum_key_given__alphabets_only(): assert schemachange.cli.get_alphanum_key("abc") == ["abc"] -def test_get_alphanum_key_upper_alphanumeric(): +def test_get_alphanum_key_given__upper_alphanumeric(): assert schemachange.cli.get_alphanum_key("V1.2.3__") == [ "v", 1, @@ -33,7 +47,7 @@ def test_get_alphanum_key_upper_alphanumeric(): ] -def test_get_alphanum_key_valid_version_string(): +def test_get_alphanum_key_given__valid_version_string(): assert schemachange.cli.get_alphanum_key("1.2.2") == ["", 1, ".", 2, ".", 2, ""] @@ -41,3 +55,81 @@ def test_sorted_alphanumeric_mixed_string(): assert schemachange.cli.sorted_alphanumeric( ["V1.2.3__file.sql", "V1.2.4__file.sql"] ) == ["V1.2.3__file.sql", "V1.2.4__file.sql"] + + +@pytest.mark.parametrize( + "cht, expected", + [ + ( + None, + { + "database_name": schemachange.cli._metadata_database_name.upper(), + "schema_name": schemachange.cli._metadata_schema_name.upper(), + "table_name": schemachange.cli._metadata_table_name.upper(), + }, + ), + ( + "change_history_table", + { + "database_name": schemachange.cli._metadata_database_name.upper(), + "schema_name": schemachange.cli._metadata_schema_name.upper(), + "table_name": "change_history_table".upper(), + }, + ), + ( + "myschema.change_history_table", + { + "database_name": schemachange.cli._metadata_database_name.upper(), + "schema_name": "myschema".upper(), + "table_name": "change_history_table".upper(), + }, + ), + ( + "mydb.myschema.change_history_table", + { + "database_name": "mydb".upper(), + "schema_name": "myschema".upper(), + "table_name": "change_history_table".upper(), + }, + ), + ( + '"change-history-table"', + { + "database_name": schemachange.cli._metadata_database_name.upper(), + "schema_name": schemachange.cli._metadata_schema_name.upper(), + "table_name": '"change-history-table"', + }, + ), + ( + '"my-schema"."change-history-table"', + { + "database_name": schemachange.cli._metadata_database_name.upper(), + "schema_name": '"my-schema"', + "table_name": '"change-history-table"', + }, + ), + ( + '"my-db"."my-schema"."change-history-table"', + { + "database_name": '"my-db"', + "schema_name": '"my-schema"', + "table_name": '"change-history-table"', + }, + ), + ], +) +def test_get_change_history_table_details_given__acceptable_values_produces_fully_qualified_change_history_table_name( + cht, expected +): + assert schemachange.cli.get_change_history_table_details(cht) == expected + +@pytest.mark.parametrize( + "cht", [("fifth.fourth.third.two.one"), ("fourth.third.two.one")] +) +def test_get_change_history_table_details_given__unacceptable_values_raises_error( + cht +): + with pytest.raises(ValueError) as e: + schemachange.cli.get_change_history_table_details(cht) + + assert str(e.value).startswith("Invalid change history table name: ") From aae74748bd1accb6c0f5ccf7c5cb8440d4592cec Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 27 Mar 2024 23:33:35 -0400 Subject: [PATCH 13/14] reverting the ruff cache ignore because it already has contextual gitignore that ignores the entire contents of the directory. --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 1a84845a..068d6ba0 100644 --- a/.gitignore +++ b/.gitignore @@ -131,6 +131,4 @@ dmypy.json .idea # Mac files -**/.DS_Store - -.ruff_cache/ \ No newline at end of file +**/.DS_Store \ No newline at end of file From 02393a332b4c7e1092f5889c16c43494cb534e02 Mon Sep 17 00:00:00 2001 From: Tiji Mathew Date: Wed, 27 Mar 2024 23:37:04 -0400 Subject: [PATCH 14/14] pushing support for 3.11 --- .github/workflows/dependency-review.yml | 2 +- .github/workflows/pytest.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index da99d0c5..b9945082 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -9,6 +9,6 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: 'Dependency Review' uses: actions/dependency-review-action@v3 diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 96e788c4..0ba747d4 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -9,10 +9,10 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: