App.yaml for merlin server

CHANGELOG.md

@@ -9,38 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Update docker docs for new rabbitmq and redis server versions
 - Added lgtm.com Badge for README.md
 - More fixes for lgtm checks.
-- Added merlin server capabilities under merlin/server/
-- Added merlin server commands init, start, status, stop to main.py
-- Added redis.conf for default redis configuration for merlin in server/redis.conf
-- Added default configurations for merlin server command in `merlin/server/*.yaml`
-- Added documentation page docs/merlin_server.rst, docs/modules/server/configuration.rst, and docs/modules/server/commands.rst
-- Added merlin server config command for editing configuration files.
-- Added server_command.py to store command calls.
-- Added following flags to config subcommand
-  - ipaddress (Set the binded ip address of the container)
-  - port (Set the binded port of the container)
-  - user (Set the main user file for container)
-  - password (Set the main user password file for container)
-  - add-user (Add a user to the container image [outputs an associated password file for user])
-  - remove-user (Remove user from list of added users)
-  - directory (Set the directory of the merlin server container files)
-  - snapshot-seconds (Set the number of seconds elapsed before snapshot change condition is checked)
-  - snapshot-changes (Set snapshot change condition for a snapshot to be made)
-  - snapshot-file (Set the database file that the snapshot will be written to)
-  - append-mode (Set the append mode for redis)
-  - append-file (Set the name of the append only file for redis)
-- Added user_file to merlin server config
-- Added pass_file to merlin server config
-- Added add_user function to add user to exisiting merlin server instance if one is running
-- Added remove_user function to remove user from merlin server instance if one is running
-- Added masteruser in redis config
-- Added requirepass in redis config
-- Added server_util.py file to store utility functions.
-- Created RedisConfig class to interface with redis.conf file
-- Created RedisUsers class to interface with redis.user file
-- Added better interface for configuration files(ServerConfig, ContainerConfig, ContainerFormatConfig, and ProcessConfig) with getting configuration values from merlin server config file, with classes.
-- Added merlin server to reapply users based on the saved redis.users config file.
-- Added redis.pass file containing password for default user in main merlin configuration.
+- Added merlin server command as a container option for broker and results_backend servers.
 - Added the flux_exec batch argument to allow for flux exec arguments,
   e.g. flux_exec: flux exec -r "0-1" to run celery workers only on
   ranks 0 and 1 of a multi-rank allocation

merlin/server/merlin_server.yaml

@@ -1,6 +1,8 @@
 container:
     # Select the format for the recipe e.g. singularity, docker, podman (currently singularity is the only working option.)
     format: singularity
+    #Type of container that is used
+    image_type: redis
     # The image name
     image: redis_latest.sif
     # The url to pull the image from

merlin/server/server_commands.py

@@ -18,7 +18,7 @@
     pull_server_config,
     pull_server_image,
 )
-from merlin.server.server_util import RedisConfig, RedisUsers
+from merlin.server.server_util import AppYaml, RedisConfig, RedisUsers
 
 
 LOG = logging.getLogger("merlin")
@@ -195,6 +195,14 @@ def start_server() -> bool:
     redis_config = RedisConfig(server_config.container.get_config_path())
     redis_users.apply_to_redis(redis_config.get_ip_address(), redis_config.get_port(), redis_config.get_password())
 
+    new_app_yaml = os.path.join(server_config.container.get_config_dir(), "app.yaml")
+    ay = AppYaml()
+    ay.apply_server_config(server_config=server_config)
+    ay.write(new_app_yaml)
+    LOG.info(f"New app.yaml written to {new_app_yaml}.")
+    LOG.info("Replace app.yaml in ~/.merlin/app.yaml to use merlin server as main configuration.")
+    LOG.info("To use for local runs, move app.yaml into the running directory.")
+
     return True
 
 

merlin/server/server_util.py

@@ -5,6 +5,8 @@
 import redis
 import yaml
 
+import merlin.utils
+
 
 LOG = logging.getLogger("merlin")
 
@@ -57,6 +59,7 @@ class ContainerConfig:
 
     # Default values for configuration
     FORMAT = "singularity"
+    IMAGE_TYPE = "redis"
     IMAGE_NAME = "redis_latest.sif"
     REDIS_URL = "docker://redis"
     CONFIG_FILE = "redis.conf"
@@ -66,6 +69,7 @@ class ContainerConfig:
     USERS_FILE = "redis.users"
 
     format = FORMAT
+    image_type = IMAGE_TYPE
     image = IMAGE_NAME
     url = REDIS_URL
     config = CONFIG_FILE
@@ -76,6 +80,7 @@ class ContainerConfig:
 
     def __init__(self, data: dict) -> None:
         self.format = data["format"] if "format" in data else self.FORMAT
+        self.image_type = data["image_type"] if "image_type" in data else self.IMAGE_TYPE
         self.image = data["image"] if "image" in data else self.IMAGE_NAME
         self.url = data["url"] if "url" in data else self.REDIS_URL
         self.config = data["config"] if "config" in data else self.CONFIG_FILE
@@ -87,6 +92,9 @@ def __init__(self, data: dict) -> None:
     def get_format(self) -> str:
         return self.format
 
+    def get_image_type(self) -> str:
+        return self.image_type
+
     def get_image_name(self) -> str:
         return self.image
 
@@ -508,3 +516,43 @@ def apply_to_redis(self, host: str, port: int, password: str) -> None:
         for user in current_users:
             if user not in self.users:
                 db.acl_deluser(user)
+
+
+class AppYaml:
+    """
+    AppYaml allows for an structured way to interact with any app.yaml main merlin configuration file.
+    It helps to parse each component of the app.yaml and allow users to edit, configure and write the
+    file.
+    """
+
+    default_filename = os.path.join(MERLIN_CONFIG_DIR, "app.yaml")
+    data = {}
+    broker_name = "broker"
+    results_name = "results_backend"
+
+    def __init__(self, filename: str = default_filename) -> None:
+        if not os.path.exists(filename):
+            filename = self.default_filename
+        self.read(filename)
+
+    def apply_server_config(self, server_config: ServerConfig):
+        rc = RedisConfig(server_config.container.get_config_path())
+
+        self.data[self.broker_name]["name"] = server_config.container.get_image_type()
+        self.data[self.broker_name]["username"] = os.environ.get("USER")
+        self.data[self.broker_name]["password"] = server_config.container.get_pass_file_path()
+        self.data[self.broker_name]["server"] = rc.get_ip_address()
+        self.data[self.broker_name]["port"] = rc.get_port()
+
+        self.data[self.results_name]["name"] = server_config.container.get_image_type()
+        self.data[self.results_name]["username"] = os.environ.get("USER")
+        self.data[self.results_name]["password"] = server_config.container.get_pass_file_path()
+        self.data[self.results_name]["server"] = rc.get_ip_address()
+        self.data[self.results_name]["port"] = rc.get_port()
+
+    def read(self, filename: str = default_filename):
+        self.data = merlin.utils.load_yaml(filename)
+
+    def write(self, filename: str = default_filename):
+        with open(filename, "w+") as f:
+            yaml.dump(self.data, f, yaml.Dumper)