From 7a1ad251c6bee6d1df7aa4bde28e89e90004d8db Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Thu, 7 Jul 2022 13:34:13 -0700 Subject: [PATCH 1/9] Added AppYaml class to pull app.yaml and make changes required for merlin server configuration --- merlin/server/server_util.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 0f8569d17..26a24dea8 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -508,3 +508,37 @@ def apply_to_redis(self, host: str, port: int, password: str) -> None: for user in current_users: if user not in self.users: db.acl_deluser(user) + +class AppYaml: + default_filename = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") + data = {} + broker_name = "broker" + results_name = "results_backend" + + def __init__(self, filename:str = default_filename) -> None: + if not os.path.exists(filename): + filename = self.default_filename + self.read(filename) + + def apply_server_config(self, server_config:ServerConfig): + rc = RedisConfig(server_config.container.get_config_path()) + + self.data[self.broker_name]["name"] = "redis" + self.data[self.broker_name]["username"] = os.environ.get("USER") + self.data[self.broker_name]["password"] = server_config.container.get_pass_file_path() + self.data[self.broker_name]["server"] = rc.get_ip_address() + self.data[self.broker_name]["port"] = rc.get_port() + + self.data[self.results_name]["name"] = "redis" + self.data[self.results_name]["username"] = os.environ.get("USER") + self.data[self.results_name]["password"] = server_config.container.get_pass_file_path() + self.data[self.results_name]["server"] = rc.get_ip_address() + self.data[self.results_name]["port"] = rc.get_port() + + def read(self, filename:str = default_filename): + with open(filename, "r") as f: + self.data = yaml.load(f, yaml.Loader) + + def write(self, filename:str = default_filename): + with open(filename, "w+") as f: + yaml.dump(self.data, f, yaml.Dumper) \ No newline at end of file From 65d726b3f512a9c5a6300eb14b6fa67bf6b4e75b Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 12 Jul 2022 12:34:11 -0700 Subject: [PATCH 2/9] Applied AppYaml class and added log message to inform users to use new app.yaml to use merlin server --- merlin/server/merlin_server.yaml | 2 ++ merlin/server/server_commands.py | 9 ++++++++- merlin/server/server_util.py | 25 ++++++++++++++++--------- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/merlin/server/merlin_server.yaml b/merlin/server/merlin_server.yaml index c351a06d4..5f9b25367 100644 --- a/merlin/server/merlin_server.yaml +++ b/merlin/server/merlin_server.yaml @@ -1,6 +1,8 @@ container: # Select the format for the recipe e.g. singularity, docker, podman (currently singularity is the only working option.) format: singularity + #Type of container that is used + image_type: redis # The image name image: redis_latest.sif # The url to pull the image from diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 5782183a4..fd3892cce 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -18,7 +18,7 @@ pull_server_config, pull_server_image, ) -from merlin.server.server_util import RedisConfig, RedisUsers +from merlin.server.server_util import AppYaml, RedisConfig, RedisUsers LOG = logging.getLogger("merlin") @@ -195,6 +195,13 @@ def start_server() -> bool: redis_config = RedisConfig(server_config.container.get_config_path()) redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password()) + new_app_yaml = os.path.join(server_config.container.get_config_dir(), "app.yaml") + ay = AppYaml() + ay.apply_server_config(server_config=server_config) + ay.write(new_app_yaml) + LOG.info(f"New app.yaml written to {new_app_yaml}.") + LOG.info(f"Replace app.yaml in ~/.merlin/app.yaml to use merlin server.") + return True diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 26a24dea8..3bce2f8cd 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -57,6 +57,7 @@ class ContainerConfig: # Default values for configuration FORMAT = "singularity" + IMAGE_TYPE = "redis" IMAGE_NAME = "redis_latest.sif" REDIS_URL = "docker://redis" CONFIG_FILE = "redis.conf" @@ -66,6 +67,7 @@ class ContainerConfig: USERS_FILE = "redis.users" format = FORMAT + image_type = IMAGE_TYPE image = IMAGE_NAME url = REDIS_URL config = CONFIG_FILE @@ -76,6 +78,7 @@ class ContainerConfig: def __init__(self, data: dict) -> None: self.format = data["format"] if "format" in data else self.FORMAT + self.type = data["image_type"] if "image_type" in data else self.IMAGE_TYPE self.image = data["image"] if "image" in data else self.IMAGE_NAME self.url = data["url"] if "url" in data else self.REDIS_URL self.config = data["config"] if "config" in data else self.CONFIG_FILE @@ -87,6 +90,9 @@ def __init__(self, data: dict) -> None: def get_format(self) -> str: return self.format + def get_image_type(self) -> str: + return self.type + def get_image_name(self) -> str: return self.image @@ -509,36 +515,37 @@ def apply_to_redis(self, host: str, port: int, password: str) -> None: if user not in self.users: db.acl_deluser(user) + class AppYaml: default_filename = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") data = {} broker_name = "broker" results_name = "results_backend" - def __init__(self, filename:str = default_filename) -> None: + def __init__(self, filename: str = default_filename) -> None: if not os.path.exists(filename): filename = self.default_filename self.read(filename) - - def apply_server_config(self, server_config:ServerConfig): + + def apply_server_config(self, server_config: ServerConfig): rc = RedisConfig(server_config.container.get_config_path()) - self.data[self.broker_name]["name"] = "redis" + self.data[self.broker_name]["name"] = server_config.container.get_image_type() self.data[self.broker_name]["username"] = os.environ.get("USER") self.data[self.broker_name]["password"] = server_config.container.get_pass_file_path() self.data[self.broker_name]["server"] = rc.get_ip_address() self.data[self.broker_name]["port"] = rc.get_port() - - self.data[self.results_name]["name"] = "redis" + + self.data[self.results_name]["name"] = server_config.container.get_image_type() self.data[self.results_name]["username"] = os.environ.get("USER") self.data[self.results_name]["password"] = server_config.container.get_pass_file_path() self.data[self.results_name]["server"] = rc.get_ip_address() self.data[self.results_name]["port"] = rc.get_port() - def read(self, filename:str = default_filename): + def read(self, filename: str = default_filename): with open(filename, "r") as f: self.data = yaml.load(f, yaml.Loader) - def write(self, filename:str = default_filename): + def write(self, filename: str = default_filename): with open(filename, "w+") as f: - yaml.dump(self.data, f, yaml.Dumper) \ No newline at end of file + yaml.dump(self.data, f, yaml.Dumper) From e95eafb177815cfd7bb7b70733d0a32b157fa513 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 12 Jul 2022 12:37:49 -0700 Subject: [PATCH 3/9] Lint changes and updated CHANGELOG --- CHANGELOG.md | 1 + merlin/server/server_commands.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 195483640..63673676a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation +- Added AppYaml class to interface with the main app.yaml config file. ### Changed - Rename lgtm.yml to .lgtm.yml - Changed "default" user password to be "merlin_password" as default. diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index fd3892cce..2ba0a071f 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -200,7 +200,7 @@ def start_server() -> bool: ay.apply_server_config(server_config=server_config) ay.write(new_app_yaml) LOG.info(f"New app.yaml written to {new_app_yaml}.") - LOG.info(f"Replace app.yaml in ~/.merlin/app.yaml to use merlin server.") + LOG.info("Replace app.yaml in ~/.merlin/app.yaml to use merlin server.") return True From ee6ab1fcbeb7dbff27de7ba7a5fc06fcf88f0667 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 12 Jul 2022 13:14:26 -0700 Subject: [PATCH 4/9] Update LOG messages to inform users regarding local runs --- merlin/server/server_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 2ba0a071f..412dad7c7 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -200,7 +200,8 @@ def start_server() -> bool: ay.apply_server_config(server_config=server_config) ay.write(new_app_yaml) LOG.info(f"New app.yaml written to {new_app_yaml}.") - LOG.info("Replace app.yaml in ~/.merlin/app.yaml to use merlin server.") + LOG.info("Replace app.yaml in ~/.merlin/app.yaml to use merlin server as main configuration.") + LOG.info("To use if for local runs move app.yaml into the running directory.") return True From 5661a7ab49741bb3e0e7186571cc1a32ba6e5401 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Tue, 12 Jul 2022 13:23:13 -0700 Subject: [PATCH 5/9] Updated LOG message to instruct users of how to use app.yaml for local configuration --- merlin/server/server_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merlin/server/server_commands.py b/merlin/server/server_commands.py index 412dad7c7..a487f4ca9 100644 --- a/merlin/server/server_commands.py +++ b/merlin/server/server_commands.py @@ -201,7 +201,7 @@ def start_server() -> bool: ay.write(new_app_yaml) LOG.info(f"New app.yaml written to {new_app_yaml}.") LOG.info("Replace app.yaml in ~/.merlin/app.yaml to use merlin server as main configuration.") - LOG.info("To use if for local runs move app.yaml into the running directory.") + LOG.info("To use for local runs, move app.yaml into the running directory.") return True From 1d31c2b4ceddfdad862080fd26bb15af2065ee8d Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Fri, 22 Jul 2022 11:45:24 -0700 Subject: [PATCH 6/9] Changed type to image type in ContainerConfig --- merlin/server/server_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index 3bce2f8cd..b6b756297 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -78,7 +78,7 @@ class ContainerConfig: def __init__(self, data: dict) -> None: self.format = data["format"] if "format" in data else self.FORMAT - self.type = data["image_type"] if "image_type" in data else self.IMAGE_TYPE + self.image_type = data["image_type"] if "image_type" in data else self.IMAGE_TYPE self.image = data["image"] if "image" in data else self.IMAGE_NAME self.url = data["url"] if "url" in data else self.REDIS_URL self.config = data["config"] if "config" in data else self.CONFIG_FILE @@ -91,7 +91,7 @@ def get_format(self) -> str: return self.format def get_image_type(self) -> str: - return self.type + return self.image_type def get_image_name(self) -> str: return self.image From 384e9afae767fd91ac21f93bc3baf1776d71fa0a Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Fri, 22 Jul 2022 11:48:09 -0700 Subject: [PATCH 7/9] Added comments for AppYaml --- merlin/server/server_util.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index b6b756297..ca79e6441 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -517,6 +517,11 @@ def apply_to_redis(self, host: str, port: int, password: str) -> None: class AppYaml: + """ + AppYaml allows for an structured way to interact with any app.yaml main merlin configuration file. + It helps to parse each component of the app.yaml and allow users to edit, configure and write the + file. + """ default_filename = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") data = {} broker_name = "broker" From bc705936dbc12d5fd57aec5a97c72ad7ab1980c8 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Fri, 22 Jul 2022 11:53:10 -0700 Subject: [PATCH 8/9] Shorten CHANGELOG.md for merlin server changes --- CHANGELOG.md | 34 +--------------------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63673676a..9e9a11d7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,42 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Update docker docs for new rabbitmq and redis server versions - Added lgtm.com Badge for README.md - More fixes for lgtm checks. -- Added merlin server capabilities under merlin/server/ -- Added merlin server commands init, start, status, stop to main.py -- Added redis.conf for default redis configuration for merlin in server/redis.conf -- Added default configurations for merlin server command in merlin/server/*.yaml -- Added documentation page docs/merlin_server.rst, docs/modules/server/configuration.rst, and docs/modules/server/commands.rst -- Added merlin server config command for editing configuration files. -- Added server_command.py to store command calls. -- Added following flags to config subcommand - - ipaddress (Set the binded ip address of the container) - - port (Set the binded port of the container) - - user (Set the main user file for container) - - password (Set the main user password file for container) - - add-user (Add a user to the container image [outputs an associated password file for user]) - - remove-user (Remove user from list of added users) - - directory (Set the directory of the merlin server container files) - - snapshot-seconds (Set the number of seconds elapsed before snapshot change condition is checked) - - snapshot-changes (Set snapshot change condition for a snapshot to be made) - - snapshot-file (Set the database file that the snapshot will be written to) - - append-mode (Set the append mode for redis) - - append-file (Set the name of the append only file for redis) -- Added user_file to merlin server config -- Added pass_file to merlin server config -- Added add_user function to add user to exisiting merlin server instance if one is running -- Added remove_user function to remove user from merlin server instance if one is running -- Added masteruser in redis config -- Added requirepass in redis config -- Added server_util.py file to store utility functions. -- Created RedisConfig class to interface with redis.conf file -- Created RedisUsers class to interface with redis.user file -- Added better interface for configuration files(ServerConfig, ContainerConfig, ContainerFormatConfig, and ProcessConfig) with getting configuration values from merlin server config file, with classes. -- Added merlin server to reapply users based on the saved redis.users config file. -- Added redis.pass file containing password for default user in main merlin configuration. +- Added merlin server command as a container option for broker and results_backend servers. - Added the flux_exec batch argument to allow for flux exec arguments, e.g. flux_exec: flux exec -r "0-1" to run celery workers only on ranks 0 and 1 of a multi-rank allocation -- Added AppYaml class to interface with the main app.yaml config file. ### Changed - Rename lgtm.yml to .lgtm.yml - Changed "default" user password to be "merlin_password" as default. From 22205cbea006153e33db5287868f3186be2224f7 Mon Sep 17 00:00:00 2001 From: Ryan Lee Date: Fri, 22 Jul 2022 11:55:46 -0700 Subject: [PATCH 9/9] Updated read in AppYaml to utilize merlin.util.load_yaml --- merlin/server/server_util.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/merlin/server/server_util.py b/merlin/server/server_util.py index ca79e6441..90c612ef0 100644 --- a/merlin/server/server_util.py +++ b/merlin/server/server_util.py @@ -5,6 +5,8 @@ import redis import yaml +import merlin.utils + LOG = logging.getLogger("merlin") @@ -519,9 +521,10 @@ def apply_to_redis(self, host: str, port: int, password: str) -> None: class AppYaml: """ AppYaml allows for an structured way to interact with any app.yaml main merlin configuration file. - It helps to parse each component of the app.yaml and allow users to edit, configure and write the + It helps to parse each component of the app.yaml and allow users to edit, configure and write the file. """ + default_filename = os.path.join(MERLIN_CONFIG_DIR, "app.yaml") data = {} broker_name = "broker" @@ -548,8 +551,7 @@ def apply_server_config(self, server_config: ServerConfig): self.data[self.results_name]["port"] = rc.get_port() def read(self, filename: str = default_filename): - with open(filename, "r") as f: - self.data = yaml.load(f, yaml.Loader) + self.data = merlin.utils.load_yaml(filename) def write(self, filename: str = default_filename): with open(filename, "w+") as f: