Back merge of fixes from 0.6.0 to 0.3.1 (#1071)

CHANGELOG-0.3.md

@@ -5,8 +5,12 @@
 ### Fixed
 
 - [#839](https://github.com/epiphany-platform/epiphany/issues/839) - Add ServerAliveInterval option to keep SSH connection for long running tasks
+- [#1016](https://github.com/epiphany-platform/epiphany/issues/1016) - Disable verify, backup and recovery as they are not fully implemented
+- [#1105](https://github.com/epiphany-platform/epiphany/issues/1105) - [v0.3] failing on task: Install container-selinux for RHEL
+- [#1106](https://github.com/epiphany-platform/epiphany/issues/1106) - [v0.3] Epicli exiting with exit code 0 despite errors
 - [#1154](https://github.com/epiphany-platform/epiphany/issues/1154) - Node exporter is not installed on logging vms
 - [#1163](https://github.com/epiphany-platform/epiphany/issues/1163) - [v0.4, v0.3] Typo in the role name rabbitmq
+- Added default machines for rabbitmq and load_balancer components
 
 ## [0.3.0] 2019-07-31
 

CHANGELOG.md

@@ -1,4 +1,4 @@
-# Development release
+# Changelog
 
 All notable changes to this project will be documented in linked files.
 
@@ -7,24 +7,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 Reference for actual cluster component versions can be found [here](docs/home/COMPONENTS.md)
 
-## Current release
-
-### 0.3.x
-
-- [CHANGELOG-0.3.1](./CHANGELOG-0.3.md#030-2020-03-xx)
-- [CHANGELOG-0.3.0](./CHANGELOG-0.3.md#030-2019-07-31)
-
-## Older releases
-
-### 0.2.x
-
-- [CHANGELOG-0.2.3](./CHANGELOG-0.2.md#023-2019-05-20)
-- [CHANGELOG-0.2.2](./CHANGELOG-0.2.md#022-2019-03-29)
-- [CHANGELOG-0.2.1](./CHANGELOG-0.2.md#021-2019-03-07)
-- [CHANGELOG-0.2.0](./CHANGELOG-0.2.md#020-2019-02-19)
-
-### 0.1.x
-
-- [CHANGELOG-0.1.2](./CHANGELOG-0.1.md#012-2018-12-05)
-- [CHANGELOG-0.1.1](./CHANGELOG-0.1.md#011-2018-11-21)
-- [CHANGELOG-0.1.0](./CHANGELOG-0.1.md#010-2018-10-19)
+- [CHANGELOG-0.3.x](./CHANGELOG-0.3.md)
+- [CHANGELOG-0.2.x](./CHANGELOG-0.2.md)
+- [CHANGELOG-0.1.x](./CHANGELOG-0.1.md)

core/src/epicli/.devcontainer/Dockerfile

@@ -0,0 +1,35 @@
+FROM python:3.7
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+ARG USERNAME=vscode
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+RUN apt-get update \
+    && apt-get -y install --no-install-recommends apt-utils dialog 2>&1 \
+
+    && apt-get -y install git procps lsb-release gcc make musl-dev libffi-dev tar unzip \
+
+    && apt-get -y install ruby-full \
+
+    && gem install serverspec rake rspec_junit_formatter \
+
+    && pip --disable-pip-version-check --no-cache-dir install pylint \
+
+    && pip --disable-pip-version-check --no-cache-dir install pipenv \
+
+    && groupadd --gid $USER_GID $USERNAME \
+    && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \
+
+    && apt-get install -y sudo \
+    && echo $USERNAME ALL=\(root\) NOPASSWD:ALL > /etc/sudoers.d/$USERNAME \
+    && chmod 0440 /etc/sudoers.d/$USERNAME \
+
+    && apt-get autoremove -y \
+    && apt-get clean -y \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV DEBIAN_FRONTEND=
+
+

core/src/epicli/.devcontainer/devcontainer.json

@@ -0,0 +1,18 @@
+{
+	"name": "epicli",
+	"dockerFile": "Dockerfile",
+	"extensions": [
+		"ms-python.python",
+		"littlefoxteam.vscode-python-test-adapter",
+		"vscoss.vscode-ansible",
+		"wholroyd.jinja",
+		"redhat.vscode-yaml",
+		"mauve.terraform",
+		"davidanson.vscode-markdownlint"
+	],
+	"settings": {
+		"python.pythonPath": "/usr/local/bin/python"
+	},
+	"postCreateCommand": "sudo --preserve-env=https_proxy pipenv install --system --dev",
+	"runArgs": [ "-u", "vscode" ]
+}

core/src/epicli/.vscode/extensions.json

@@ -0,0 +1,12 @@
+{
+    "recommendations": [
+        "ms-vscode-remote.remote-containers",
+        "ms-python.python",
+        "littlefoxteam.vscode-python-test-adapter",
+        "vscoss.vscode-ansible",
+        "wholroyd.jinja",
+        "redhat.vscode-yaml",
+        "mauve.terraform",
+        "davidanson.vscode-markdownlint"
+    ]
+}

core/src/epicli/.vscode/launch.json

@@ -0,0 +1,21 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "epicli",
+            "type": "python",
+            "request": "launch",
+            "program": "${workspaceFolder}/cli/epicli.py",
+            "cwd": "${workspaceFolder}",
+            "pythonPath": "${config:python.pythonPath}",
+            "env": { "PYTHONPATH": "${workspaceFolder}" },
+            "console": "integratedTerminal",
+            "args": ["apply", "-f", "${workspaceFolder}/clusters/YOUR_DATA_YAML.yml"]
+            // "args": ["delete", "-b", "${workspaceFolder}/clusters/build/dir"]
+            // "args": ["init", "-p", "PROVIDER", "-n", "NAME"]
+        }
+    ]	
+}

core/src/epicli/.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "python.linting.enabled": true,
+    "python.testing.unittestEnabled": false,
+    "python.testing.nosetestsEnabled": false,
+    "python.testing.pytestEnabled": true,
+}

core/src/epicli/cli/engine/AnsibleCommand.py

@@ -47,6 +47,8 @@ def run_task_with_retries(self, inventory, module, args, hosts, retries, timeout
                 self.logger.error(e)
                 self.logger.info("Retry running task: " + str(i + 1) + "/" + str(retries))
                 time.sleep(timeout)
+        else:
+            raise Exception(f'Failed running task after {str(retries)} retries')
 
     def run_playbook(self, inventory, playbook_path):
         cmd = ['ansible-playbook']
@@ -78,4 +80,5 @@ def run_playbook_with_retries(self, inventory, playbook_path, retries, timeout=1
                 self.logger.error(e)
                 self.logger.info("Retry running playbook: " + str(i + 1) + "/" + str(retries))
                 time.sleep(timeout)
-        return 1
+        else:
+            raise Exception(f'Failed running playbook after {str(retries)} retries')

core/src/epicli/cli/engine/EpiphanyEngine.py

@@ -77,51 +77,43 @@ def collect_infrastructure_config(self):
             config_collector.run()
 
     def verify(self):
-        try:
-            self.process_input_docs()
+        self.process_input_docs()
 
-            self.process_configuration_docs()
+        self.process_configuration_docs()
 
-            self.process_infrastructure_docs()
+        self.process_infrastructure_docs()
 
-            save_manifest([*self.input_docs, *self.configuration_docs, *self.infrastructure_docs], self.cluster_model.specification.name)
+        save_manifest([*self.input_docs, *self.configuration_docs, *self.infrastructure_docs], self.cluster_model.specification.name)
 
-            return 0
-        except Exception as e:
-            self.logger.error(e, exc_info=True) #TODO extensive debug output might not always be wanted. Make this configurable with input flag?
-            return 1
+        return 0
 
     def apply(self):
-        try:
-            self.process_input_docs()
+        self.process_input_docs()
 
-            self.process_infrastructure_docs()
+        self.process_infrastructure_docs()
 
-            if not self.skip_infrastructure:
-                # Generate terraform templates
-                with TerraformTemplateGenerator(self.cluster_model, self.infrastructure_docs) as template_generator:
-                    template_generator.run()
+        if not self.skip_infrastructure:
+            # Generate terraform templates
+            with TerraformTemplateGenerator(self.cluster_model, self.infrastructure_docs) as template_generator:
+                template_generator.run()
 
-                # Run Terraform to create infrastructure
-                with TerraformRunner(self.cluster_model.specification.name) as tf_runner:
-                    tf_runner.run()
+            # Run Terraform to create infrastructure
+            with TerraformRunner(self.cluster_model.specification.name) as tf_runner:
+                tf_runner.run()
 
-            self.process_configuration_docs()
+        self.process_configuration_docs()
 
-            self.collect_infrastructure_config()
+        self.collect_infrastructure_config()
 
-            # Run Ansible to provision infrastructure
-            docs = [*self.input_docs, *self.configuration_docs, *self.infrastructure_docs]
-            with AnsibleRunner(self.cluster_model, docs) as ansible_runner:
-                ansible_runner.run()
+        # Run Ansible to provision infrastructure
+        docs = [*self.input_docs, *self.configuration_docs, *self.infrastructure_docs]
+        with AnsibleRunner(self.cluster_model, docs) as ansible_runner:
+            ansible_runner.run()
 
-            # Save docs to manifest file
-            save_manifest(docs, self.cluster_model.specification.name)
+        # Save docs to manifest file
+        save_manifest(docs, self.cluster_model.specification.name)
 
-            return 0
-        except Exception as e:
-            self.logger.error(e, exc_info=True)  # TODO extensive debug output might not always be wanted. Make this configurable with input flag?
-            return 1
+        return 0
 
     def dry_run(self):
 

core/src/epicli/cli/engine/PatchEngine.py

@@ -24,56 +24,45 @@ def run(self):
         pass
 
     def run_upgrade(self):
-        try:
-            build_directory = Config().output_dir
-            build_roles_directory = os.path.join(build_directory, 'ansible/roles')
+        build_directory = Config().output_dir
+        build_roles_directory = os.path.join(build_directory, 'ansible/roles')
 
-            upgrade_playbook_path = os.path.join(build_roles_directory, 'upgrade')
-            backup_playbook_path = os.path.join(build_roles_directory, 'backup')
-            recovery_playbook_path = os.path.join(build_roles_directory, 'recovery')
+        upgrade_playbook_path = os.path.join(build_roles_directory, 'upgrade')
+        backup_playbook_path = os.path.join(build_roles_directory, 'backup')
+        recovery_playbook_path = os.path.join(build_roles_directory, 'recovery')
 
-            upgrade_role_path = os.path.join(build_directory, 'ansible', 'upgrade.yml')
+        upgrade_role_path = os.path.join(build_directory, 'ansible', 'upgrade.yml')
 
-            epiphany_playbooks_path = os.path.dirname(__file__) + AnsibleRunner.ANSIBLE_PLAYBOOKS_PATH
-            epiphany_roles_path = os.path.join(epiphany_playbooks_path, 'roles')
+        epiphany_playbooks_path = os.path.dirname(__file__) + AnsibleRunner.ANSIBLE_PLAYBOOKS_PATH
+        epiphany_roles_path = os.path.join(epiphany_playbooks_path, 'roles')
 
-            upgrade_role_source_path = os.path.join(epiphany_roles_path, 'upgrade')
-            backup_role_source_path = os.path.join(epiphany_roles_path, 'backup')
-            restore_role_source_path = os.path.join(epiphany_roles_path, 'recovery')
-            playbook_source_path = os.path.join(epiphany_playbooks_path, 'upgrade.yml')
+        upgrade_role_source_path = os.path.join(epiphany_roles_path, 'upgrade')
+        backup_role_source_path = os.path.join(epiphany_roles_path, 'backup')
+        restore_role_source_path = os.path.join(epiphany_roles_path, 'recovery')
+        playbook_source_path = os.path.join(epiphany_playbooks_path, 'upgrade.yml')
 
-            copy_files_recursively(upgrade_role_source_path, upgrade_playbook_path)
-            copy_files_recursively(backup_role_source_path, backup_playbook_path)
-            copy_files_recursively(restore_role_source_path, recovery_playbook_path)
-            copy_file(playbook_source_path, upgrade_role_path)
+        copy_files_recursively(upgrade_role_source_path, upgrade_playbook_path)
+        copy_files_recursively(backup_role_source_path, backup_playbook_path)
+        copy_files_recursively(restore_role_source_path, recovery_playbook_path)
+        copy_file(playbook_source_path, upgrade_role_path)
 
-            inventory_path = get_inventory_path_for_build(build_directory)
-            self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=upgrade_role_path)
-            return 0
-        except Exception as e:
-            self.logger.error(e, exc_info=True)  # TODO extensive debug output might not always be wanted. Make this configurable with input flag?
-            return 1
+        inventory_path = get_inventory_path_for_build(build_directory)
+        self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=upgrade_role_path)
+
+        return 0
 
     def run_backup(self):
-        try:
-            build_directory = Config().output_dir
-            backup_role_path = os.path.join(build_directory, 'ansible', 'backup.yml')
-            inventory_path = get_inventory_path_for_build(build_directory)
-            self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=backup_role_path)
+        build_directory = Config().output_dir
+        backup_role_path = os.path.join(build_directory, 'ansible', 'backup.yml')
+        inventory_path = get_inventory_path_for_build(build_directory)
+        self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=backup_role_path)
 
-            return 0
-        except Exception as e:
-            self.logger.error(e, exc_info=True)  # TODO extensive debug output might not always be wanted. Make this configurable with input flag?
-            return 1
+        return 0
 
     def run_recovery(self):
-        try:
-            build_directory = Config().output_dir
-            backup_role_path = os.path.join(build_directory, 'ansible', 'recovery.yml')
-            inventory_path = get_inventory_path_for_build(build_directory)
-            self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=backup_role_path)
-
-            return 0
-        except Exception as e:
-            self.logger.error(e, exc_info=True)  # TODO extensive debug output might not always be wanted. Make this configurable with input flag?
-            return 1
+        build_directory = Config().output_dir
+        backup_role_path = os.path.join(build_directory, 'ansible', 'recovery.yml')
+        inventory_path = get_inventory_path_for_build(build_directory)
+        self.ansible_command.run_playbook(inventory=inventory_path, playbook_path=backup_role_path)
+
+        return 0

core/src/epicli/cli/engine/UserConfigInitializer.py

@@ -22,20 +22,16 @@ def __exit__(self, exc_type, exc_value, traceback):
         super().__exit__(exc_type, exc_value, traceback)
 
     def run(self):
-        try:
-            defaults = load_all_yaml_objs(types.DEFAULT, self.provider, 'configuration/minimal-cluster-config')
-            defaults[0].specification.name = self.name
+        defaults = load_all_yaml_objs(types.DEFAULT, self.provider, 'configuration/minimal-cluster-config')
+        defaults[0].specification.name = self.name
 
-            if self.is_full_config:
-                defaults = self.get_full_config(defaults)
+        if self.is_full_config:
+            defaults = self.get_full_config(defaults)
 
-            save_manifest(defaults, self.name, self.name+'.yml')
+        save_manifest(defaults, self.name, self.name+'.yml')
 
-            self.logger.info('Initialized user configuration and saved it to "' + os.path.join(get_build_path(self.name), self.name + '.yml') + '"')
-            return 0
-        except Exception as e:
-            self.logger.error(e, exc_info=True)  # TODO extensive debug output might not always be wanted. Make this configurable with input flag?
-            return 1
+        self.logger.info('Initialized user configuration and saved it to "' + os.path.join(get_build_path(self.name), self.name + '.yml') + '"')
+        return 0
 
     def get_full_config(self, config_docs):
         cluster_config_path = save_manifest(config_docs, self.name, self.name + '.yml')