From adc5922d7bbf9b09033bd468658604394d198d49 Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Sat, 21 May 2016 22:47:25 +0200 Subject: [PATCH 01/14] Create plugin_manager and plugin class Signed-off-by: Alexander Schneider --- .gitignore | 1 + compose/cli/main.py | 38 ++++++++++++++++++- compose/cli/utils.py | 13 +++++++ compose/const.py | 2 + compose/plugin.py | 20 ++++++++++ compose/plugin_manager.py | 77 +++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 +- 7 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 compose/plugin.py create mode 100644 compose/plugin_manager.py diff --git a/.gitignore b/.gitignore index 4b318e2328c..19e1368faaf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.idea *.egg-info *.pyc .coverage* diff --git a/compose/cli/main.py b/compose/cli/main.py index 34e7f35c764..dfb6a052e6c 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -42,6 +42,9 @@ from .log_printer import LogPrinter from .utils import get_version_info from .utils import yesno +from .utils import get_plugin_dir +from ..plugin_manager import PluginManager +from ..plugin_manager import NoPluginError if not IS_WINDOWS_PLATFORM: @@ -103,6 +106,11 @@ def perform_command(options, handler, command_options): handler(command, options, command_options) return + if options['COMMAND'] == 'plugin': + command = TopLevelCommand(None) + handler(command, command_options) + return + project = project_from_options('.', options) command = TopLevelCommand(project) with errors.handle_connection_errors(project.client): @@ -173,6 +181,7 @@ class TopLevelCommand(object): kill Kill containers logs View output from containers pause Pause services + plugin Manages the plugins port Print the public port for a port binding ps List containers pull Pulls service images @@ -596,7 +605,7 @@ def run(self, options): if options['--publish'] and options['--service-ports']: raise UserError( 'Service port mapping and manual port mapping ' - 'can not be used togather' + 'can not be used together' ) if options['COMMAND']: @@ -776,6 +785,33 @@ def version(cls, options): else: print(get_version_info('full')) + def plugin(self, options): + """ + Manages docker-compose plugins + + Usage: plugin [install|uninstall|list|config] [PLUGIN] + """ + + plugin_manager = PluginManager(get_plugin_dir()) + + if options['list']: + plugins = plugin_manager.get_plugins() + + for plugin_name, plugin in plugins.items(): + print(plugin.name, plugin.description) + elif options['PLUGIN'] is not None: + try: + if options['install']: + plugin_manager.install_plugin(options['PLUGIN']) + elif options['uninstall']: + plugin_manager.uninstall_plugin(options['PLUGIN']) + elif options['config']: + plugin_manager.configure_plugin(options['PLUGIN']) + except NoPluginError: + raise UserError("Plugin %s doesn't exist" % options['PLUGIN']) + else: + return False + def convergence_strategy_from_opts(options): no_recreate = options['--no-recreate'] diff --git a/compose/cli/utils.py b/compose/cli/utils.py index fff4a543f4c..32741a48e26 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -12,6 +12,11 @@ import compose + +from ..const import HOME_DIR +from ..const import PLUGIN_DIR + + # WindowsError is not defined on non-win32 platforms. Avoid runtime errors by # defining it as OSError (its parent class) if missing. try: @@ -93,3 +98,11 @@ def get_build_version(): with open(filename) as fh: return fh.read().strip() + + +def get_user_home(): + return os.path.expanduser("~") + + +def get_plugin_dir(): + return os.path.join(get_user_home(), HOME_DIR, PLUGIN_DIR) diff --git a/compose/const.py b/compose/const.py index 9e00d96e9fd..983d205d82d 100644 --- a/compose/const.py +++ b/compose/const.py @@ -14,6 +14,8 @@ LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' +HOME_DIR = '.docker-compose' +PLUGIN_DIR = 'plugins' COMPOSEFILE_V1 = '1' COMPOSEFILE_V2_0 = '2.0' diff --git a/compose/plugin.py b/compose/plugin.py new file mode 100644 index 00000000000..37c1f911677 --- /dev/null +++ b/compose/plugin.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import +from __future__ import unicode_literals +import os +import inspect + +class Plugin(object): + def __init__(self): + self.path = os.path.abspath(inspect.getfile(self.__class__)) + print(self.path) + self.name = os.path.dirname(self.path) + self.description = '' + + def install(self): + return True + + def uninstall(self): + return True + + def configure(self): + return True diff --git a/compose/plugin_manager.py b/compose/plugin_manager.py new file mode 100644 index 00000000000..5d55d276356 --- /dev/null +++ b/compose/plugin_manager.py @@ -0,0 +1,77 @@ +# import importlib.util +from __future__ import absolute_import +from __future__ import unicode_literals + +from .plugin import Plugin # needed for building +import os +import shutil +import sys +import imp + + +class NoPluginError(Exception): + pass + + +class PluginManager(object): + def __init__( + self, + plugin_dir + ): + self.plugin_dir = plugin_dir + self.plugin_list = {} + + for current_plugin_dir in os.listdir(self.plugin_dir): + dir_with_path = os.path.join(self.plugin_dir, current_plugin_dir) + + if os.path.isdir(dir_with_path): + init_file = os.path.join(dir_with_path, '__init__.py') + + if not os.path.isfile(init_file): + raise NoPluginError("Plugin at '{}' is has no ini file".format(dir_with_path)) + + sys.path.append(dir_with_path) + plugin_instance = imp.load_source(current_plugin_dir, init_file) + + ''' + spec = importlib.util.spec_from_file_location(current_plugin_dir, init_file) + plugin_instance = importlib.util.module_from_spec(spec) + spec.loader.exec_module(plugin_instance) + ''' + + if not hasattr(plugin_instance, 'plugin'): + raise NoPluginError("Plugin at '{}' is not a plugin".format(current_plugin_dir)) + + if not isinstance(plugin_instance.plugin, Plugin): + raise NoPluginError("Plugin at '{}' is not a plugin".format(current_plugin_dir)) + + self.plugin_list[current_plugin_dir] = plugin_instance.plugin + + def get_plugins(self): + return self.plugin_list + + def __plugin_exists(self, name): + if name not in self.plugin_list: + raise NoPluginError("Plugin {} doesn't exists".format(name)) + + def is_plugin_installed(self, name): + self.__plugin_exists(name) + return self.plugin_list[name].is_installed() + + + def install_plugin(self, name): + self.__plugin_exists(name) + return self.plugin_list[name].install() + + def uninstall_plugin(self, name): + self.__plugin_exists(name) + + # After the uninstall method was successful remove the plugin + if self.plugin_list[name].uninstall(): + return shutil.rmtree(self.plugin_list[name].path) + + return False + + def configure_plugin(self, name): + self.__plugin_exists(name) + return self.plugin_list[name].configure() diff --git a/requirements.txt b/requirements.txt index eb5275f4e19..a4a1d837fbe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.2.0 docker-py==1.8.1 dockerpty==0.4.1 docopt==0.6.1 -enum34==1.0.4 +enum34==1.1.5 jsonschema==2.5.1 requests==2.7.0 six==1.7.3 From d0cdd23ef66b7944e92da9e41ef4ee5403935dfb Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Wed, 1 Jun 2016 13:30:41 +0200 Subject: [PATCH 02/14] Add test Refactor code Adjust config schema Signed-off-by: Alexander Schneider --- tests/unit/plugin_manager_test.py | 0 tests/unit/plugin_test.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/unit/plugin_manager_test.py create mode 100644 tests/unit/plugin_test.py diff --git a/tests/unit/plugin_manager_test.py b/tests/unit/plugin_manager_test.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/plugin_test.py b/tests/unit/plugin_test.py new file mode 100644 index 00000000000..e69de29bb2d From 3b6f51cbab2f19f4b7cd5254cf18e41f3a488a4c Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Wed, 1 Jun 2016 13:31:43 +0200 Subject: [PATCH 03/14] Add missing files for commit Signed-off-by: Alexander Schneider --- compose/cli/main.py | 61 +++++++---- compose/cli/utils.py | 2 - compose/config/config.py | 17 ++- compose/config/config_schema_v2.0.json | 20 ++++ compose/config/serialize.py | 1 + compose/plugin.py | 58 ++++++++-- compose/plugin_manager.py | 142 +++++++++++++++++++------ script/test/versions.py | 11 +- tests/unit/plugin_test.py | 22 ++++ 9 files changed, 266 insertions(+), 68 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index dfb6a052e6c..d31feafba9e 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -21,6 +21,8 @@ from ..config.serialize import serialize_config from ..const import DEFAULT_TIMEOUT from ..const import IS_WINDOWS_PLATFORM +from ..plugin import PluginError +from ..plugin_manager import PluginManager from ..progress_stream import StreamOutputError from ..project import NoSuchService from ..project import OneOffFilter @@ -40,12 +42,9 @@ from .formatter import Formatter from .log_printer import build_log_presenters from .log_printer import LogPrinter +from .utils import get_plugin_dir from .utils import get_version_info from .utils import yesno -from .utils import get_plugin_dir -from ..plugin_manager import PluginManager -from ..plugin_manager import NoPluginError - if not IS_WINDOWS_PLATFORM: from dockerpty.pty import PseudoTerminal, RunOperation, ExecOperation @@ -74,6 +73,9 @@ def main(): except NeedsBuildError as e: log.error("Service '%s' needs to be built, but --no-build was passed." % e.service.name) sys.exit(1) + except PluginError as e: + log.error(e.msg) + sys.exit(1) except errors.ConnectionError: sys.exit(1) @@ -199,6 +201,7 @@ class TopLevelCommand(object): def __init__(self, project, project_dir='.'): self.project = project self.project_dir = '.' + self.plugin_manager = PluginManager(get_plugin_dir()) def build(self, options): """ @@ -792,25 +795,41 @@ def plugin(self, options): Usage: plugin [install|uninstall|list|config] [PLUGIN] """ - plugin_manager = PluginManager(get_plugin_dir()) - if options['list']: - plugins = plugin_manager.get_plugins() - - for plugin_name, plugin in plugins.items(): - print(plugin.name, plugin.description) - elif options['PLUGIN'] is not None: - try: - if options['install']: - plugin_manager.install_plugin(options['PLUGIN']) - elif options['uninstall']: - plugin_manager.uninstall_plugin(options['PLUGIN']) - elif options['config']: - plugin_manager.configure_plugin(options['PLUGIN']) - except NoPluginError: - raise UserError("Plugin %s doesn't exist" % options['PLUGIN']) + plugins = self.plugin_manager.get_plugins() + + if len(plugins) <= 0: + print('No plugins installed.') + else: + headers = [ + 'Name', + 'Description', + 'Version' + ] + rows = [] + + for plugin_name, plugin in plugins.items(): + rows.append([ + plugin.name, + plugin.description, + plugin.version + ]) + + print(Formatter().table(headers, rows)) + + elif options['install']: + self.plugin_manager.install_plugin(options['PLUGIN']) + elif options['uninstall']: + if self.plugin_manager.is_plugin_installed(options['PLUGIN']): + print("Going to remove plugin '{}'".format(options['PLUGIN'])) + + if options.get('--force') or yesno("Are you sure? [yN] ", default=False): + self.plugin_manager.uninstall_plugin(options['PLUGIN']) + elif options['config']: + self.plugin_manager.configure_plugin(options['PLUGIN']) else: - return False + subject = get_handler(self, 'plugin') + print(getdoc(subject)) def convergence_strategy_from_opts(options): diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 32741a48e26..b627d124763 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -11,8 +11,6 @@ from six.moves import input import compose - - from ..const import HOME_DIR from ..const import PLUGIN_DIR diff --git a/compose/config/config.py b/compose/config/config.py index e52de4bf8d3..943eb513b2f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -190,8 +190,11 @@ def get_volumes(self): def get_networks(self): return {} if self.version == V1 else self.config.get('networks', {}) + def get_plugins(self): + return {} if self.version == V1 else self.config.get('plugins', {}) -class Config(namedtuple('_Config', 'version services volumes networks')): + +class Config(namedtuple('_Config', 'version services volumes networks plugins')): """ :param version: configuration version :type version: int @@ -201,6 +204,8 @@ class Config(namedtuple('_Config', 'version services volumes networks')): :type volumes: :class:`dict` :param networks: Dictionary mapping network names to description dictionaries :type networks: :class:`dict` + :param plugins: Dictionary mapping plugin names to description dictionaries + :type plugins: :class:`dict` """ @@ -316,13 +321,16 @@ def load(config_details): networks = load_mapping( config_details.config_files, 'get_networks', 'Network' ) + plugins = load_mapping( + config_details.config_files, 'get_plugins', 'Plugins' + ) service_dicts = load_services(config_details, main_file) if main_file.version != V1: for service_dict in service_dicts: match_named_volumes(service_dict, volumes) - return Config(main_file.version, service_dicts, volumes, networks) + return Config(main_file.version, service_dicts, volumes, networks, plugins) def load_mapping(config_files, get_func, entity_type): @@ -434,6 +442,11 @@ def process_config_file(config_file, environment, service_name=None): config_file.get_networks(), 'network', environment,) + processed_config['plugins'] = interpolate_config_section( + config_file.filename, + config_file.get_plugins(), + 'plugin', + environment,) if config_file.version == V1: processed_config = services diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index e84d13179f6..83c83895c24 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -38,6 +38,17 @@ } }, "additionalProperties": false + }, + + "plugins": { + "id": "#/properties/plugins", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/plugin" + } + }, + "additionalProperties": false } }, @@ -313,6 +324,15 @@ } } } + }, + + "plugin": { + "version": { + "type": "string" + }, + "id": "#/definitions/plugin", + "type": "object", + "options": {"$ref": "#/definitions/list_or_dict"} } } } diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 1b498c01633..56d4d810436 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -33,6 +33,7 @@ def serialize_config(config): 'services': services, 'networks': config.networks, 'volumes': config.volumes, + 'plugins': config.plugins, } return yaml.safe_dump( diff --git a/compose/plugin.py b/compose/plugin.py index 37c1f911677..d4352aa90b1 100644 --- a/compose/plugin.py +++ b/compose/plugin.py @@ -1,20 +1,64 @@ from __future__ import absolute_import from __future__ import unicode_literals -import os + import inspect +import json +import os + + +class PluginError(Exception): + def __init__(self, *message, **errors): + # Call the base class constructor with the parameters it needs + super(PluginError, self).__init__(message, errors) + + self.message = self.__get_message() + + def __get_message(self): + return self.message + + +class PluginJsonFileError(PluginError): + pass + class Plugin(object): + required_fields = ['name', 'version'] + def __init__(self): - self.path = os.path.abspath(inspect.getfile(self.__class__)) - print(self.path) - self.name = os.path.dirname(self.path) + file = os.path.abspath(inspect.getfile(self.__class__)) + self.path = os.path.dirname(file) + self.name = os.path.basename(self.path) self.description = '' + self.version = None + + plugin_file = os.path.join(self.path, 'plugin.json') + self.load_plugin_info_from_file(plugin_file) + + @staticmethod + def check_required_plugin_file_settings(plugin_info, required_keys): + for required_key in required_keys: + if required_key not in plugin_info: + raise PluginJsonFileError("Missing json attribute '{}'".format(required_key)) + + def load_plugin_info_from_file(self, file): + if os.path.isfile(file): + plugin_info = json.load(open(file)) + + self.check_required_plugin_file_settings(plugin_info, self.required_fields) + self.name = plugin_info['name'] + self.description = plugin_info['description'] if 'description' in plugin_info else '' + self.version = plugin_info['version'] + else: + raise PluginJsonFileError('JSON plugin file not found') def install(self): - return True + pass def uninstall(self): - return True + pass + + def update(self): + pass def configure(self): - return True + print("'{}' needs no configuration".format(self.name)) diff --git a/compose/plugin_manager.py b/compose/plugin_manager.py index 5d55d276356..c216da533ae 100644 --- a/compose/plugin_manager.py +++ b/compose/plugin_manager.py @@ -1,67 +1,135 @@ -# import importlib.util from __future__ import absolute_import from __future__ import unicode_literals -from .plugin import Plugin # needed for building +import imp import os import shutil import sys -import imp +import tarfile +import urllib.request as request +import zipfile + +from .plugin import Plugin +from .plugin import PluginError + + +class PluginDoesNotExistError(PluginError): + pass -class NoPluginError(Exception): +class InvalidPluginError(PluginError): + pass + + +class InvalidPluginFileTypeError(PluginError): pass class PluginManager(object): - def __init__( - self, - plugin_dir - ): + def __init__(self, plugin_dir): self.plugin_dir = plugin_dir + self.__plugin_download_dir = os.path.join(self.plugin_dir, '.downloads') self.plugin_list = {} - for current_plugin_dir in os.listdir(self.plugin_dir): - dir_with_path = os.path.join(self.plugin_dir, current_plugin_dir) - - if os.path.isdir(dir_with_path): - init_file = os.path.join(dir_with_path, '__init__.py') + if os.path.isdir(plugin_dir): + for current_plugin_dir in os.listdir(self.plugin_dir): + plugin_path = os.path.join(self.plugin_dir, current_plugin_dir) - if not os.path.isfile(init_file): - raise NoPluginError("Plugin at '{}' is has no ini file".format(dir_with_path)) + if os.path.isdir(plugin_path): + try: + self.__load_plugin(plugin_path) + except InvalidPluginError: + print("Invalid plugin '{}' installed".format(current_plugin_dir)) - sys.path.append(dir_with_path) - plugin_instance = imp.load_source(current_plugin_dir, init_file) + def __load_plugin(self, path): + current_plugin_dir = os.path.basename(path) + init_file = os.path.join(path, '__init__.py') - ''' - spec = importlib.util.spec_from_file_location(current_plugin_dir, init_file) - plugin_instance = importlib.util.module_from_spec(spec) - spec.loader.exec_module(plugin_instance) - ''' + if not os.path.isfile(init_file): + raise InvalidPluginError( + "Missing __init__.py file." + ) - if not hasattr(plugin_instance, 'plugin'): - raise NoPluginError("Plugin at '{}' is not a plugin".format(current_plugin_dir)) + sys.path.append(path) # doesn't work with pyinstaller :( + plugin_instance = imp.load_source(current_plugin_dir, init_file) - if not isinstance(plugin_instance.plugin, Plugin): - raise NoPluginError("Plugin at '{}' is not a plugin".format(current_plugin_dir)) + if not hasattr(plugin_instance, 'plugin'): + raise InvalidPluginError( + "Plugin '{}' is not a plugin. Missing plugin attribute.".format(current_plugin_dir) + ) - self.plugin_list[current_plugin_dir] = plugin_instance.plugin + if not isinstance(plugin_instance.plugin, Plugin): + raise InvalidPluginError( + "Wrong plugin instance.".format(current_plugin_dir) + ) - def get_plugins(self): - return self.plugin_list + self.plugin_list[current_plugin_dir] = plugin_instance.plugin + return self.plugin_list[current_plugin_dir] def __plugin_exists(self, name): if name not in self.plugin_list: - raise NoPluginError("Plugin {} doesn't exists".format(name)) + raise PluginDoesNotExistError("Plugin '{}' doesn't exists".format(name)) + + def get_plugins(self): + return self.plugin_list def is_plugin_installed(self, name): - self.__plugin_exists(name) - return self.plugin_list[name].is_installed() + try: + self.__plugin_exists(name) + return True + except PluginDoesNotExistError: + return False + def __get_plugin_file(self, plugin): + try: + file = os.path.join(self.__plugin_download_dir, os.path.basename(plugin)) + request.urlretrieve(plugin, file) + except ValueError: # invalid URL + file = os.path.realpath(plugin) - def install_plugin(self, name): - self.__plugin_exists(name) - return self.plugin_list[name].install() + if not os.path.isfile(file): + return False + + return file + + def __check_plugin_archive(self, file): + if zipfile.is_zipfile(file): + archive = zipfile.ZipFile(file) + elif tarfile.is_tarfile(file): + archive = tarfile.TarFile(file) + else: + raise InvalidPluginFileTypeError('Invalid file type.') + + plugin_folder = None + + # TODO better check + for file in archive.namelist(): + if file.endswith('plugin.json'): + plugin_folder = os.path.dirname(file) + break + + if plugin_folder is None: + raise InvalidPluginFileTypeError('Missing plugin.json file.') + + archive.extractall(self.plugin_dir) + return os.path.join(self.plugin_dir, plugin_folder) + + def install_plugin(self, plugin): + file = self.__get_plugin_file(plugin) + + if not os.path.isdir(self.plugin_dir): + os.makedirs(self.plugin_dir) + + plugin_path = self.__check_plugin_archive(file) + + try: + self.__load_plugin(plugin_path).install() + except InvalidPluginError as e: + shutil.rmtree(plugin_path) + raise e + + if os.path.isdir(self.__plugin_download_dir): + shutil.rmtree(self.__plugin_download_dir) def uninstall_plugin(self, name): self.__plugin_exists(name) @@ -72,6 +140,10 @@ def uninstall_plugin(self, name): return False + def update_plugin(self, name): + self.__plugin_exists(name) + return self.plugin_list[name].update() + def configure_plugin(self, name): self.__plugin_exists(name) return self.plugin_list[name].configure() diff --git a/script/test/versions.py b/script/test/versions.py index 98f97ef3243..25290c055d5 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -28,6 +28,7 @@ import argparse import itertools import operator +import re from collections import namedtuple import requests @@ -40,8 +41,16 @@ class Version(namedtuple('_Version', 'major minor patch rc')): @classmethod def parse(cls, version): - version = version.lstrip('v') + version = re.search( + r'v((\d+\.)?(\d+\.)?(\*|\d+)(-rc(\d+))?)', + version + ).group(1) + version, _, rc = version.partition('-') + + if version.count('.') is 1: + version += '.0' + major, minor, patch = version.split('.', 3) return cls(int(major), int(minor), int(patch), rc) diff --git a/tests/unit/plugin_test.py b/tests/unit/plugin_test.py index e69de29bb2d..dd8e8531c0f 100644 --- a/tests/unit/plugin_test.py +++ b/tests/unit/plugin_test.py @@ -0,0 +1,22 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import pytest + +from .. import mock +from .. import unittest +from compose.plugin import Plugin +from compose.plugin import PluginJsonFileError + + +class PluginTest(unittest.TestCase): + def setUp(self): + self.invalid_plugin_info = {'var1': 'value1'} + + @mock.patch('compose.plugin.os') + def test_load_plugin_info_from_file(self, mock_os): + pass + + def test_check_required_plugin_file_settings(self): + with pytest.raises(PluginJsonFileError): + Plugin.check_required_plugin_file_settings(self.invalid_plugin_info, Plugin.required_fields) From 04f2af5d5c4a5b6d7feaff86db32ff4265a494f5 Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Mon, 4 Jul 2016 00:22:59 +0200 Subject: [PATCH 04/14] Adjust existing tests with new plugin config parameter Write plugin class tests Make sure that all existing tests pass Signed-off-by: Alexander Schneider --- compose/plugin.py | 25 ++++++++++++++++------- compose/plugin_manager.py | 14 ++++++++----- tests/integration/service_test.py | 2 +- tests/unit/plugin_test.py | 34 ++++++++++++++++++++++++++----- tests/unit/project_test.py | 12 +++++++++++ 5 files changed, 69 insertions(+), 18 deletions(-) diff --git a/compose/plugin.py b/compose/plugin.py index d4352aa90b1..fff9a79f300 100644 --- a/compose/plugin.py +++ b/compose/plugin.py @@ -11,7 +11,7 @@ def __init__(self, *message, **errors): # Call the base class constructor with the parameters it needs super(PluginError, self).__init__(message, errors) - self.message = self.__get_message() + self.message = message def __get_message(self): return self.message @@ -21,10 +21,15 @@ class PluginJsonFileError(PluginError): pass +class PluginNotImplementError(PluginError): + pass + + class Plugin(object): required_fields = ['name', 'version'] - def __init__(self): + def __init__(self, config=None): + self.config = config file = os.path.abspath(inspect.getfile(self.__class__)) self.path = os.path.dirname(file) self.name = os.path.basename(self.path) @@ -40,14 +45,17 @@ def check_required_plugin_file_settings(plugin_info, required_keys): if required_key not in plugin_info: raise PluginJsonFileError("Missing json attribute '{}'".format(required_key)) + return True + def load_plugin_info_from_file(self, file): if os.path.isfile(file): - plugin_info = json.load(open(file)) + with open(file) as f: + plugin_info = json.load(f) - self.check_required_plugin_file_settings(plugin_info, self.required_fields) - self.name = plugin_info['name'] - self.description = plugin_info['description'] if 'description' in plugin_info else '' - self.version = plugin_info['version'] + self.check_required_plugin_file_settings(plugin_info, self.required_fields) + self.name = plugin_info['name'] + self.description = plugin_info['description'] if 'description' in plugin_info else '' + self.version = plugin_info['version'] else: raise PluginJsonFileError('JSON plugin file not found') @@ -62,3 +70,6 @@ def update(self): def configure(self): print("'{}' needs no configuration".format(self.name)) + + def execute(self): + raise PluginNotImplementError("Method execute for '{}' must be implemented".format(self.name)) diff --git a/compose/plugin_manager.py b/compose/plugin_manager.py index c216da533ae..73eefbbbb91 100644 --- a/compose/plugin_manager.py +++ b/compose/plugin_manager.py @@ -4,9 +4,13 @@ import imp import os import shutil -import sys import tarfile -import urllib.request as request + +try: + import urllib.request as request +except ImportError: + import urllib2 as request + import zipfile from .plugin import Plugin @@ -50,7 +54,6 @@ def __load_plugin(self, path): "Missing __init__.py file." ) - sys.path.append(path) # doesn't work with pyinstaller :( plugin_instance = imp.load_source(current_plugin_dir, init_file) if not hasattr(plugin_instance, 'plugin'): @@ -58,12 +61,13 @@ def __load_plugin(self, path): "Plugin '{}' is not a plugin. Missing plugin attribute.".format(current_plugin_dir) ) - if not isinstance(plugin_instance.plugin, Plugin): + if not issubclass(plugin_instance.plugin, Plugin): raise InvalidPluginError( "Wrong plugin instance.".format(current_plugin_dir) ) - self.plugin_list[current_plugin_dir] = plugin_instance.plugin + self.plugin_list[current_plugin_dir] = plugin_instance.plugin({}) # TODO add config + return self.plugin_list[current_plugin_dir] def __plugin_exists(self, name): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index df50d513a9e..1801f5bfc78 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -397,7 +397,7 @@ def test_execute_convergence_plan_when_host_volume_is_removed(self): assert not mock_log.warn.called assert ( - [mount['Destination'] for mount in new_container.get('Mounts')], + [mount['Destination'] for mount in new_container.get('Mounts')] == ['/data'] ) assert new_container.get_mount('/data')['Source'] != host_path diff --git a/tests/unit/plugin_test.py b/tests/unit/plugin_test.py index dd8e8531c0f..a1d5c612d71 100644 --- a/tests/unit/plugin_test.py +++ b/tests/unit/plugin_test.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +import json + import pytest from .. import mock @@ -10,13 +12,35 @@ class PluginTest(unittest.TestCase): - def setUp(self): - self.invalid_plugin_info = {'var1': 'value1'} + valid_plugin_info = {'name': 'Plugin name', 'version': '1.0.0', 'description': 'Plugin description'} + invalid_plugin_info = {'var1': 'value1'} - @mock.patch('compose.plugin.os') - def test_load_plugin_info_from_file(self, mock_os): + def setUp(self): pass - def test_check_required_plugin_file_settings(self): + @mock.patch('compose.plugin.os.path.isfile') + def test_load_plugin_info_from_file(self, mock_isfile): + mock_isfile.return_value = True + json_file = json.dumps(self.valid_plugin_info, False, True) + m = mock.mock_open(read_data=json_file) + + with mock.patch('compose.plugin.open', m, create=True): + Plugin.__init__ = mock.Mock(return_value=None) + plugin = Plugin() + plugin.load_plugin_info_from_file('plugin.json') + + assert plugin.name == 'Plugin name' + assert plugin.version == '1.0.0' + assert plugin.description == 'Plugin description' + + def test_check_required_plugin_file_settings_error(self): with pytest.raises(PluginJsonFileError): Plugin.check_required_plugin_file_settings(self.invalid_plugin_info, Plugin.required_fields) + + def test_check_required_plugin_file_settings_success(self): + check = Plugin.check_required_plugin_file_settings( + self.valid_plugin_info, + Plugin.required_fields + ) + + assert check is True diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index b6a52e08d05..dec98413f22 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -36,6 +36,7 @@ def test_from_config(self): ], networks=None, volumes=None, + plugins=None, ) project = Project.from_config( name='composetest', @@ -64,6 +65,7 @@ def test_from_config_v2(self): ], networks=None, volumes=None, + plugins=None, ) project = Project.from_config('composetest', config, None) self.assertEqual(len(project.services), 2) @@ -170,6 +172,7 @@ def test_use_volumes_from_container(self): }], networks=None, volumes=None, + plugins=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"] @@ -202,6 +205,7 @@ def test_use_volumes_from_service_no_container(self): ], networks=None, volumes=None, + plugins=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"] @@ -227,6 +231,7 @@ def test_use_volumes_from_service_container(self): ], networks=None, volumes=None, + plugins=None, ), ) with mock.patch.object(Service, 'containers') as mock_return: @@ -360,6 +365,7 @@ def test_net_unset(self): ], networks=None, volumes=None, + plugins=None, ), ) service = project.get_service('test') @@ -384,6 +390,7 @@ def test_use_net_from_container(self): ], networks=None, volumes=None, + plugins=None, ), ) service = project.get_service('test') @@ -417,6 +424,7 @@ def test_use_net_from_service(self): ], networks=None, volumes=None, + plugins=None, ), ) @@ -437,6 +445,7 @@ def test_uses_default_network_true(self): ], networks=None, volumes=None, + plugins=None, ), ) @@ -457,6 +466,7 @@ def test_uses_default_network_false(self): ], networks={'custom': {}}, volumes=None, + plugins=None, ), ) @@ -487,6 +497,7 @@ def test_container_without_name(self): }], networks=None, volumes=None, + plugins=None, ), ) self.assertEqual([c.id for c in project.containers()], ['1']) @@ -503,6 +514,7 @@ def test_down_with_no_resources(self): }], networks={'default': {}}, volumes={'data': {}}, + plugins=None, ), ) self.mock_client.remove_network.side_effect = NotFound(None, None, 'oops') From 2fe1813114517b552fa46a9eb9b3ba400a1004c2 Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Tue, 19 Jul 2016 00:13:23 +0200 Subject: [PATCH 05/14] Load plugin manager at the right time Add decorators for adding commands and patching functions Remove redundant functions Signed-off-by: Alexander Schneider --- compose/cli/main.py | 27 ++++++++----- compose/plugin.py | 84 ++++++++++++++++++++++++++++++++++++--- compose/plugin_manager.py | 34 +++++++++------- tests/unit/plugin_test.py | 2 +- 4 files changed, 117 insertions(+), 30 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index d31feafba9e..fd6430806ca 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -81,6 +81,8 @@ def main(): def dispatch(): + plugin_manager = PluginManager(get_plugin_dir()) + plugin_manager.get_plugins() # TODO setup_logging() dispatcher = DocoptDispatcher( TopLevelCommand, @@ -94,27 +96,34 @@ def dispatch(): sys.exit(1) setup_console_handler(console_handler, options.get('--verbose')) - return functools.partial(perform_command, options, handler, command_options) + return functools.partial(perform_command, options, handler, command_options, plugin_manager) -def perform_command(options, handler, command_options): +def perform_command(options, handler, command_options, plugin_manager): if options['COMMAND'] in ('help', 'version'): # Skip looking up the compose file. handler(command_options) return if options['COMMAND'] == 'config': - command = TopLevelCommand(None) + command = TopLevelCommand(None, plugin_manager) handler(command, options, command_options) return - if options['COMMAND'] == 'plugin': - command = TopLevelCommand(None) + none_project_commands = ['plugin'] + + for tlc_command in TopLevelCommand.__dict__: + if hasattr(getattr(TopLevelCommand, tlc_command), '__standalone__') and \ + getattr(TopLevelCommand, tlc_command).__standalone__ is True: + none_project_commands.append(tlc_command) + + if options['COMMAND'] in none_project_commands: + command = TopLevelCommand(None, plugin_manager) handler(command, command_options) return project = project_from_options('.', options) - command = TopLevelCommand(project) + command = TopLevelCommand(project, plugin_manager) with errors.handle_connection_errors(project.client): handler(command, command_options) @@ -198,10 +207,10 @@ class TopLevelCommand(object): version Show the Docker-Compose version information """ - def __init__(self, project, project_dir='.'): + def __init__(self, project, plugin_manager, project_dir='.'): self.project = project - self.project_dir = '.' - self.plugin_manager = PluginManager(get_plugin_dir()) + self.plugin_manager = plugin_manager + self.project_dir = project_dir def build(self, options): """ diff --git a/compose/plugin.py b/compose/plugin.py index fff9a79f300..c14c115e96a 100644 --- a/compose/plugin.py +++ b/compose/plugin.py @@ -1,9 +1,75 @@ from __future__ import absolute_import from __future__ import unicode_literals +import collections import inspect import json import os +import re +from abc import ABCMeta +from abc import abstractmethod +from functools import partial + +import compose + + +def compose_patch(obj, name): + def wrapper(fnc): + original = getattr(obj, name) + + if fnc.__doc__ is None: + fnc.__doc__ = original.__doc__ + + method = partial(fnc, original) + method.__doc__ = fnc.__doc__ + setattr(obj, name, method) + + return fnc + return wrapper + + +def compose_command(standalone=False): + def update_command_doc(original_doc, fnc_name, fnc_doc): + pre_doc = '' + command_regex = r'(\s*)([^ ]+)(\s*)(.*)' + doc_commands = None + + for compose_doc_line in original_doc.splitlines(): + if doc_commands is not None and re.match(command_regex, compose_doc_line): + command = re.search(command_regex, compose_doc_line) + doc_commands[command.group(2)] = compose_doc_line + + if fnc_name not in doc_commands: + space_to_text = len(command.group(2) + command.group(3)) + new_command = command.group(1) + fnc_name + new_command += (' ' * (space_to_text - len(fnc_name))) + new_command += fnc_doc.strip(' \t\n\r').splitlines()[0] + doc_commands[fnc_name] = new_command + else: + if re.match(r'\s*Commands:\s*', compose_doc_line): + doc_commands = {} + + pre_doc += compose_doc_line + '\n' + + doc_commands = collections.OrderedDict(sorted(doc_commands.items())) + return pre_doc + '\n'.join(doc_commands.values()) + + def wrap(fnc): + def return_fnc(*args, **kargs): + raise PluginCommandError( + "Command function '{}' must not called out of scope.".format(fnc.__name__) + ) + + fnc.__standalone__ = standalone + compose.cli.main.TopLevelCommand.__doc__ = update_command_doc( + compose.cli.main.TopLevelCommand.__doc__, + fnc.__name__, + fnc.__doc__ + ) + + setattr(compose.cli.main.TopLevelCommand, fnc.__name__, fnc) + return return_fnc + return wrap class PluginError(Exception): @@ -25,16 +91,23 @@ class PluginNotImplementError(PluginError): pass -class Plugin(object): +class PluginCommandError(PluginError): + pass + + +class Plugin: + __metaclass__ = ABCMeta required_fields = ['name', 'version'] - def __init__(self, config=None): - self.config = config + def __init__(self, plugin_manager): + self.plugin_manager = plugin_manager file = os.path.abspath(inspect.getfile(self.__class__)) self.path = os.path.dirname(file) - self.name = os.path.basename(self.path) + self.id = os.path.basename(self.path) + self.name = self.id self.description = '' self.version = None + self.config = None plugin_file = os.path.join(self.path, 'plugin.json') self.load_plugin_info_from_file(plugin_file) @@ -71,5 +144,6 @@ def update(self): def configure(self): print("'{}' needs no configuration".format(self.name)) + @abstractmethod def execute(self): - raise PluginNotImplementError("Method execute for '{}' must be implemented".format(self.name)) + pass diff --git a/compose/plugin_manager.py b/compose/plugin_manager.py index 73eefbbbb91..806774ecd72 100644 --- a/compose/plugin_manager.py +++ b/compose/plugin_manager.py @@ -33,17 +33,7 @@ class PluginManager(object): def __init__(self, plugin_dir): self.plugin_dir = plugin_dir self.__plugin_download_dir = os.path.join(self.plugin_dir, '.downloads') - self.plugin_list = {} - - if os.path.isdir(plugin_dir): - for current_plugin_dir in os.listdir(self.plugin_dir): - plugin_path = os.path.join(self.plugin_dir, current_plugin_dir) - - if os.path.isdir(plugin_path): - try: - self.__load_plugin(plugin_path) - except InvalidPluginError: - print("Invalid plugin '{}' installed".format(current_plugin_dir)) + self.plugin_list = None def __load_plugin(self, path): current_plugin_dir = os.path.basename(path) @@ -66,15 +56,29 @@ def __load_plugin(self, path): "Wrong plugin instance.".format(current_plugin_dir) ) - self.plugin_list[current_plugin_dir] = plugin_instance.plugin({}) # TODO add config - - return self.plugin_list[current_plugin_dir] + return plugin_instance.plugin(self) def __plugin_exists(self, name): - if name not in self.plugin_list: + if name not in self.get_plugins(): raise PluginDoesNotExistError("Plugin '{}' doesn't exists".format(name)) def get_plugins(self): + if self.plugin_list is None: + self.plugin_list = {} + + if os.path.isdir(self.plugin_dir): + for current_plugin_dir in os.listdir(self.plugin_dir): + plugin_path = os.path.join(self.plugin_dir, current_plugin_dir) + + if os.path.isdir(plugin_path): + try: + plugin = self.__load_plugin(plugin_path) + self.plugin_list[plugin.id] = plugin + except InvalidPluginError: + print("Invalid plugin '{}' installed".format(current_plugin_dir)) + except TypeError as e: + print("Invalid plugin error: {}".format(str(e))) + return self.plugin_list def is_plugin_installed(self, name): diff --git a/tests/unit/plugin_test.py b/tests/unit/plugin_test.py index a1d5c612d71..b28632d5ff3 100644 --- a/tests/unit/plugin_test.py +++ b/tests/unit/plugin_test.py @@ -26,7 +26,7 @@ def test_load_plugin_info_from_file(self, mock_isfile): with mock.patch('compose.plugin.open', m, create=True): Plugin.__init__ = mock.Mock(return_value=None) - plugin = Plugin() + plugin = Plugin(None) # TODO plugin_manager mock plugin.load_plugin_info_from_file('plugin.json') assert plugin.name == 'Plugin name' From 11eb8615226dbfaa07bb973812815fcc1d3cb3ed Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Sun, 31 Jul 2016 00:28:06 +0200 Subject: [PATCH 06/14] Finish first implementation Signed-off-by: Alexander Schneider --- compose/cli/docopt_command.py | 8 ++- compose/cli/main.py | 3 +- compose/plugin.py | 23 ++++--- compose/plugin_manager.py | 116 +++++++++++++++++++++++++--------- 4 files changed, 106 insertions(+), 44 deletions(-) diff --git a/compose/cli/docopt_command.py b/compose/cli/docopt_command.py index 809a4b7455e..b339310ee9c 100644 --- a/compose/cli/docopt_command.py +++ b/compose/cli/docopt_command.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +from inspect import cleandoc from inspect import getdoc from docopt import docopt @@ -21,7 +22,12 @@ def __init__(self, command_class, options): self.options = options def parse(self, argv): - command_help = getdoc(self.command_class) + # Fix for http://bugs.python.org/issue12773 + if hasattr(self.command_class, '__modified_doc__'): + command_help = cleandoc(self.command_class.__modified_doc__) + else: + command_help = getdoc(self.command_class) + options = docopt_full_help(command_help, argv, **self.options) command = options['COMMAND'] diff --git a/compose/cli/main.py b/compose/cli/main.py index fd6430806ca..2e7d123dff3 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -82,7 +82,6 @@ def main(): def dispatch(): plugin_manager = PluginManager(get_plugin_dir()) - plugin_manager.get_plugins() # TODO setup_logging() dispatcher = DocoptDispatcher( TopLevelCommand, @@ -96,6 +95,7 @@ def dispatch(): sys.exit(1) setup_console_handler(console_handler, options.get('--verbose')) + plugin_manager.load_config('.', options) return functools.partial(perform_command, options, handler, command_options, plugin_manager) @@ -825,7 +825,6 @@ def plugin(self, options): ]) print(Formatter().table(headers, rows)) - elif options['install']: self.plugin_manager.install_plugin(options['PLUGIN']) elif options['uninstall']: diff --git a/compose/plugin.py b/compose/plugin.py index c14c115e96a..33a04c3c2be 100644 --- a/compose/plugin.py +++ b/compose/plugin.py @@ -6,8 +6,6 @@ import json import os import re -from abc import ABCMeta -from abc import abstractmethod from functools import partial import compose @@ -22,8 +20,11 @@ def wrapper(fnc): method = partial(fnc, original) method.__doc__ = fnc.__doc__ - setattr(obj, name, method) + if hasattr(original, '__standalone__'): + method.__standalone__ = original.__standalone__ + + setattr(obj, name, method) return fnc return wrapper @@ -61,12 +62,18 @@ def return_fnc(*args, **kargs): ) fnc.__standalone__ = standalone - compose.cli.main.TopLevelCommand.__doc__ = update_command_doc( + modified_doc = update_command_doc( compose.cli.main.TopLevelCommand.__doc__, fnc.__name__, fnc.__doc__ ) + # Using __modified_doc__ as fix for http://bugs.python.org/issue12773 + try: + compose.cli.main.TopLevelCommand.__doc__ = modified_doc + except AttributeError: + compose.cli.main.TopLevelCommand.__modified_doc__ = modified_doc + setattr(compose.cli.main.TopLevelCommand, fnc.__name__, fnc) return return_fnc return wrap @@ -96,11 +103,11 @@ class PluginCommandError(PluginError): class Plugin: - __metaclass__ = ABCMeta required_fields = ['name', 'version'] - def __init__(self, plugin_manager): + def __init__(self, plugin_manager, config): self.plugin_manager = plugin_manager + self.config = config file = os.path.abspath(inspect.getfile(self.__class__)) self.path = os.path.dirname(file) self.id = os.path.basename(self.path) @@ -143,7 +150,3 @@ def update(self): def configure(self): print("'{}' needs no configuration".format(self.name)) - - @abstractmethod - def execute(self): - pass diff --git a/compose/plugin_manager.py b/compose/plugin_manager.py index 806774ecd72..29a5a7a1aab 100644 --- a/compose/plugin_manager.py +++ b/compose/plugin_manager.py @@ -6,6 +6,11 @@ import shutil import tarfile +from compose.cli.command import get_config_path_from_options +from compose.config import config +from compose.config.environment import Environment +from compose.config.errors import ComposeFileNotFound + try: import urllib.request as request except ImportError: @@ -29,13 +34,45 @@ class InvalidPluginFileTypeError(PluginError): pass +class NoneLoadedConfigError(PluginError): + pass + + class PluginManager(object): def __init__(self, plugin_dir): self.plugin_dir = plugin_dir self.__plugin_download_dir = os.path.join(self.plugin_dir, '.downloads') + self.config = None + self.plugin_classes = None self.plugin_list = None - def __load_plugin(self, path): + self.plugin_classes = self.__get_plugin_classes() + + def load_config(self, project_dir, options): + try: + environment = Environment.from_env_file(project_dir) + config_path = get_config_path_from_options(project_dir, options, environment) + config_details = config.find(project_dir, config_path, environment) + self.config = config.load(config_details) + except ComposeFileNotFound: + self.config = False + + self.__load_plugins() + + def __get_plugin_paths(self): + paths = {} + + if os.path.isdir(self.plugin_dir): + for current_plugin_dir in os.listdir(self.plugin_dir): + plugin_path = os.path.join(self.plugin_dir, current_plugin_dir) + + if os.path.isdir(plugin_path): + paths[current_plugin_dir] = plugin_path + + return paths + + @staticmethod + def __source_plugin(path): current_plugin_dir = os.path.basename(path) init_file = os.path.join(path, '__init__.py') @@ -44,46 +81,61 @@ def __load_plugin(self, path): "Missing __init__.py file." ) - plugin_instance = imp.load_source(current_plugin_dir, init_file) + plugin_package = imp.load_source(current_plugin_dir, init_file) - if not hasattr(plugin_instance, 'plugin'): + if not hasattr(plugin_package, 'plugin'): raise InvalidPluginError( "Plugin '{}' is not a plugin. Missing plugin attribute.".format(current_plugin_dir) ) - if not issubclass(plugin_instance.plugin, Plugin): + if not issubclass(plugin_package.plugin, Plugin): raise InvalidPluginError( "Wrong plugin instance.".format(current_plugin_dir) ) - return plugin_instance.plugin(self) + return plugin_package.plugin - def __plugin_exists(self, name): - if name not in self.get_plugins(): - raise PluginDoesNotExistError("Plugin '{}' doesn't exists".format(name)) + def __get_plugin_classes(self): + if self.plugin_classes is None: + self.plugin_classes = {} - def get_plugins(self): + for (plugin_id, plugin_path) in self.__get_plugin_paths().items(): + try: + plugin = self.__source_plugin(plugin_path) + self.plugin_classes[plugin_id] = plugin + except InvalidPluginError: + print("Invalid plugin '{}' installed".format(plugin_id)) + except TypeError as e: + print("Invalid plugin error: {}".format(str(e))) + + return self.plugin_classes + + def __load_plugins(self): if self.plugin_list is None: - self.plugin_list = {} + if self.config is None: + raise NoneLoadedConfigError("The configuration wan't loaded for the plugin manager. " + "Plugins can only instantiated after that.") - if os.path.isdir(self.plugin_dir): - for current_plugin_dir in os.listdir(self.plugin_dir): - plugin_path = os.path.join(self.plugin_dir, current_plugin_dir) + plugins_config = self.config.plugins if self.config is not False else {} + self.plugin_list = {} - if os.path.isdir(plugin_path): - try: - plugin = self.__load_plugin(plugin_path) - self.plugin_list[plugin.id] = plugin - except InvalidPluginError: - print("Invalid plugin '{}' installed".format(current_plugin_dir)) - except TypeError as e: - print("Invalid plugin error: {}".format(str(e))) + for (plugin_id, plugin_class) in self.__get_plugin_classes().items(): + plugin_config = plugins_config[plugin_id] if plugin_id in plugins_config else {} + plugin_instance = plugin_class(self, plugin_config) + self.plugin_list[plugin_id] = plugin_instance return self.plugin_list - def is_plugin_installed(self, name): + def get_plugins(self): + return self.__load_plugins() + + def __plugin_exists(self, plugin_id): + if id not in self.get_plugins(): + raise PluginDoesNotExistError("Plugin '{}' doesn't exists".format(plugin_id)) + + def is_plugin_installed(self, plugin_id): try: - self.__plugin_exists(name) + self.__plugin_exists(plugin_id) return True except PluginDoesNotExistError: return False @@ -110,7 +162,7 @@ def __check_plugin_archive(self, file): plugin_folder = None - # TODO better check + # TODO improve check for file in archive.namelist(): if file.endswith('plugin.json'): plugin_folder = os.path.dirname(file) @@ -131,7 +183,9 @@ def install_plugin(self, plugin): plugin_path = self.__check_plugin_archive(file) try: - self.__load_plugin(plugin_path).install() + plugin_class = self.__source_plugin(plugin_path) + plugin_instance = plugin_class(self, {}) + plugin_instance.install() except InvalidPluginError as e: shutil.rmtree(plugin_path) raise e @@ -148,10 +202,10 @@ def uninstall_plugin(self, name): return False - def update_plugin(self, name): - self.__plugin_exists(name) - return self.plugin_list[name].update() + def update_plugin(self, plugin_id): + self.__plugin_exists(plugin_id) + return self.plugin_list[plugin_id].update() - def configure_plugin(self, name): - self.__plugin_exists(name) - return self.plugin_list[name].configure() + def configure_plugin(self, plugin_id): + self.__plugin_exists(plugin_id) + return self.plugin_list[plugin_id].configure() From d087a6eafd5ecc1af5fd9d08c70b972a630a847e Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Thu, 25 Aug 2016 00:56:29 +0200 Subject: [PATCH 07/14] Finish unit tests Signed-off-by: Alexander Schneider --- compose/cli/main.py | 135 ++++++--- compose/plugin.py | 56 ++-- compose/plugin_manager.py | 81 +++--- tests/integration/project_test.py | 42 ++- tests/unit/bundle_test.py | 0 tests/unit/cli_test.py | 15 +- tests/unit/plugin_manager_test.py | 441 ++++++++++++++++++++++++++++++ tests/unit/plugin_test.py | 173 +++++++++++- 8 files changed, 817 insertions(+), 126 deletions(-) create mode 100644 tests/unit/bundle_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 2e7d123dff3..fa40a6f92b6 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -22,6 +22,9 @@ from ..const import DEFAULT_TIMEOUT from ..const import IS_WINDOWS_PLATFORM from ..plugin import PluginError +from ..plugin_manager import InvalidPluginError +from ..plugin_manager import InvalidPluginFileTypeError +from ..plugin_manager import PluginDoesNotExistError from ..plugin_manager import PluginManager from ..progress_stream import StreamOutputError from ..project import NoSuchService @@ -54,7 +57,12 @@ def main(): - command = dispatch() + try: + command = dispatch() + except PluginError as e: + setup_logging() + log.error("Plugin error: %s", str(e)) + sys.exit(1) try: command() @@ -460,6 +468,73 @@ def pause(self, options): containers = self.project.pause(service_names=options['SERVICE']) exit_if(not containers, 'No containers to pause', 1) + def plugin(self, options): + """ + Manages docker-compose plugins + + Usage: plugin [list|install|update|config|uninstall] [PLUGIN] + """ + + if options['list']: + plugins = self.plugin_manager.get_plugins() + + if len(plugins) <= 0: + print('No plugins installed.') + else: + headers = [ + 'ID', + 'Name', + 'Description', + 'Version' + ] + rows = [] + + for plugin_name, plugin in plugins.items(): + rows.append([ + plugin.id, + plugin.name, + plugin.description, + plugin.version + ]) + + print(Formatter().table(headers, rows)) + elif options['install']: + plugin_command( + self.plugin_manager, + 'install_plugin', + options['PLUGIN'], + "Plugin '{}' successfully installed.", + "An error occurred during the installation of plugin '{}'." + ) + elif options['uninstall']: + if self.plugin_manager.is_plugin_installed(options['PLUGIN']): + print("Going to remove plugin '{}'".format(options['PLUGIN'])) + + if options.get('--force') or yesno("Are you sure? [yN] ", default=False): + self.plugin_manager.uninstall_plugin(options['PLUGIN']) + else: + log.error("Plugin '{}' isn't installed".format(options['PLUGIN'])) + sys.exit(1) + elif options['config']: + plugin_command( + self.plugin_manager, + 'configure_plugin', + options['PLUGIN'], + "Configuration of plugin '{}' successfully.", + "An error occurred during the configuration of plugin '{}'." + ) + elif options['update']: + plugin_command( + self.plugin_manager, + 'update_plugin', + options['PLUGIN'], + "Update of plugin '{}' successfully.", + "An error occurred during the update of plugin '{}'." + ) + else: + subject = get_handler(self, 'plugin') + print(getdoc(subject)) + def port(self, options): """ Print the public port for a port binding. @@ -797,48 +872,6 @@ def version(cls, options): else: print(get_version_info('full')) - def plugin(self, options): - """ - Manages docker-compose plugins - - Usage: plugin [install|uninstall|list|config] [PLUGIN] - """ - - if options['list']: - plugins = self.plugin_manager.get_plugins() - - if len(plugins) <= 0: - print('No plugins installed.') - else: - headers = [ - 'Name', - 'Description', - 'Version' - ] - rows = [] - - for plugin_name, plugin in plugins.items(): - rows.append([ - plugin.name, - plugin.description, - plugin.version - ]) - - print(Formatter().table(headers, rows)) - elif options['install']: - self.plugin_manager.install_plugin(options['PLUGIN']) - elif options['uninstall']: - if self.plugin_manager.is_plugin_installed(options['PLUGIN']): - print("Going to remove plugin '{}'".format(options['PLUGIN'])) - - if options.get('--force') or yesno("Are you sure? [yN] ", default=False): - self.plugin_manager.uninstall_plugin(options['PLUGIN']) - elif options['config']: - self.plugin_manager.configure_plugin(options['PLUGIN']) - else: - subject = get_handler(self, 'plugin') - print(getdoc(subject)) - def convergence_strategy_from_opts(options): no_recreate = options['--no-recreate'] @@ -1015,3 +1048,19 @@ def exit_if(condition, message, exit_code): if condition: log.error(message) raise SystemExit(exit_code) + + +def plugin_command(plugin_manager, command, plugin_name, success_message, error_message): + try: + success = getattr(plugin_manager, command)(plugin_name) + except (PluginDoesNotExistError, InvalidPluginError, InvalidPluginFileTypeError) as e: + log.error(str(e)) + sys.exit(1) + + if success is True: + print(success_message.format(plugin_name)) + elif success is False: + log.error(error_message.format(plugin_name)) + sys.exit(1) + elif success is None: + print("Nothing to do.") diff --git a/compose/plugin.py b/compose/plugin.py index 33a04c3c2be..c47bc5f686b 100644 --- a/compose/plugin.py +++ b/compose/plugin.py @@ -11,20 +11,33 @@ import compose -def compose_patch(obj, name): +class PartialMethod(partial): + def __get__(self, instance, owner): + if instance is None: + return self + + return partial( + self.func, + instance, + *(self.args or ()), + **(self.keywords or {}) + ) + + +def compose_patch(scope, name): def wrapper(fnc): - original = getattr(obj, name) + original = getattr(scope, name) if fnc.__doc__ is None: fnc.__doc__ = original.__doc__ - method = partial(fnc, original) - method.__doc__ = fnc.__doc__ + patched = PartialMethod(fnc, original) + patched.__doc__ = fnc.__doc__ if hasattr(original, '__standalone__'): - method.__standalone__ = original.__standalone__ + patched.__standalone__ = original.__standalone__ - setattr(obj, name, method) + setattr(scope, name, patched) return fnc return wrapper @@ -61,14 +74,15 @@ def return_fnc(*args, **kargs): "Command function '{}' must not called out of scope.".format(fnc.__name__) ) + # Using __modified_doc__ as fix for http://bugs.python.org/issue12773 + if hasattr(compose.cli.main.TopLevelCommand, '__modified_doc__'): + original_doc = compose.cli.main.TopLevelCommand.__modified_doc__ + else: + original_doc = compose.cli.main.TopLevelCommand.__doc__ + fnc.__standalone__ = standalone - modified_doc = update_command_doc( - compose.cli.main.TopLevelCommand.__doc__, - fnc.__name__, - fnc.__doc__ - ) + modified_doc = update_command_doc(original_doc, fnc.__name__, fnc.__doc__) - # Using __modified_doc__ as fix for http://bugs.python.org/issue12773 try: compose.cli.main.TopLevelCommand.__doc__ = modified_doc except AttributeError: @@ -80,14 +94,7 @@ def return_fnc(*args, **kargs): class PluginError(Exception): - def __init__(self, *message, **errors): - # Call the base class constructor with the parameters it needs - super(PluginError, self).__init__(message, errors) - - self.message = message - - def __get_message(self): - return self.message + pass class PluginJsonFileError(PluginError): @@ -131,7 +138,6 @@ def load_plugin_info_from_file(self, file): if os.path.isfile(file): with open(file) as f: plugin_info = json.load(f) - self.check_required_plugin_file_settings(plugin_info, self.required_fields) self.name = plugin_info['name'] self.description = plugin_info['description'] if 'description' in plugin_info else '' @@ -140,13 +146,13 @@ def load_plugin_info_from_file(self, file): raise PluginJsonFileError('JSON plugin file not found') def install(self): - pass + return True def uninstall(self): - pass + return True def update(self): - pass + return None def configure(self): - print("'{}' needs no configuration".format(self.name)) + return None diff --git a/compose/plugin_manager.py b/compose/plugin_manager.py index 29a5a7a1aab..efe2bef37f5 100644 --- a/compose/plugin_manager.py +++ b/compose/plugin_manager.py @@ -4,6 +4,7 @@ import imp import os import shutil +import sys import tarfile from compose.cli.command import get_config_path_from_options @@ -14,7 +15,7 @@ try: import urllib.request as request except ImportError: - import urllib2 as request + import urllib as request import zipfile @@ -46,7 +47,7 @@ def __init__(self, plugin_dir): self.plugin_classes = None self.plugin_list = None - self.plugin_classes = self.__get_plugin_classes() + self.plugin_classes = self._get_plugin_classes() def load_config(self, project_dir, options): try: @@ -57,9 +58,9 @@ def load_config(self, project_dir, options): except ComposeFileNotFound: self.config = False - self.__load_plugins() + self._load_plugins() - def __get_plugin_paths(self): + def _get_plugin_paths(self): paths = {} if os.path.isdir(self.plugin_dir): @@ -72,7 +73,7 @@ def __get_plugin_paths(self): return paths @staticmethod - def __source_plugin(path): + def _source_plugin(path): current_plugin_dir = os.path.basename(path) init_file = os.path.join(path, '__init__.py') @@ -81,6 +82,9 @@ def __source_plugin(path): "Missing __init__.py file." ) + for root, dirs, files in os.walk(path): + sys.path.append(root) + plugin_package = imp.load_source(current_plugin_dir, init_file) if not hasattr(plugin_package, 'plugin'): @@ -95,31 +99,26 @@ def __source_plugin(path): return plugin_package.plugin - def __get_plugin_classes(self): + def _get_plugin_classes(self): if self.plugin_classes is None: self.plugin_classes = {} - for (plugin_id, plugin_path) in self.__get_plugin_paths().items(): - try: - plugin = self.__source_plugin(plugin_path) - self.plugin_classes[plugin_id] = plugin - except InvalidPluginError: - print("Invalid plugin '{}' installed".format(plugin_id)) - except TypeError as e: - print("Invalid plugin error: {}".format(str(e))) + for (plugin_id, plugin_path) in self._get_plugin_paths().items(): + plugin = self._source_plugin(plugin_path) + self.plugin_classes[plugin_id] = plugin return self.plugin_classes - def __load_plugins(self): + def _load_plugins(self): if self.plugin_list is None: if self.config is None: - raise NoneLoadedConfigError("The configuration wan't loaded for the plugin manager. " + raise NoneLoadedConfigError("The configuration wasn't loaded for the plugin manager. " "Plugins can only instantiated after that.") plugins_config = self.config.plugins if self.config is not False else {} self.plugin_list = {} - for (plugin_id, plugin_class) in self.__get_plugin_classes().items(): + for (plugin_id, plugin_class) in self._get_plugin_classes().items(): plugin_config = plugins_config[plugin_id] if plugin_id in plugins_config else {} plugin_instance = plugin_class(self, plugin_config) self.plugin_list[plugin_id] = plugin_instance @@ -127,32 +126,35 @@ def __load_plugins(self): return self.plugin_list def get_plugins(self): - return self.__load_plugins() + return self._load_plugins() - def __plugin_exists(self, plugin_id): - if id not in self.get_plugins(): + def _plugin_exists(self, plugin_id): + if plugin_id not in self.get_plugins(): raise PluginDoesNotExistError("Plugin '{}' doesn't exists".format(plugin_id)) def is_plugin_installed(self, plugin_id): try: - self.__plugin_exists(plugin_id) + self._plugin_exists(plugin_id) return True except PluginDoesNotExistError: return False - def __get_plugin_file(self, plugin): + def _get_plugin_file(self, plugin): try: + if not os.path.isdir(self.__plugin_download_dir): + os.mkdir(self.__plugin_download_dir) + file = os.path.join(self.__plugin_download_dir, os.path.basename(plugin)) request.urlretrieve(plugin, file) except ValueError: # invalid URL file = os.path.realpath(plugin) if not os.path.isfile(file): - return False + raise InvalidPluginError('Invalid plugin url or file given.') return file - def __check_plugin_archive(self, file): + def _check_plugin_archive(self, file): if zipfile.is_zipfile(file): archive = zipfile.ZipFile(file) elif tarfile.is_tarfile(file): @@ -160,32 +162,37 @@ def __check_plugin_archive(self, file): else: raise InvalidPluginFileTypeError('Invalid file type.') + root_folders = [] plugin_folder = None - # TODO improve check for file in archive.namelist(): - if file.endswith('plugin.json'): + if file.endswith('/') and file.count('/') == 1: + root_folders.append(file[:-1]) + + if file.count('/') == 1 and file.endswith('plugin.json'): plugin_folder = os.path.dirname(file) - break - if plugin_folder is None: - raise InvalidPluginFileTypeError('Missing plugin.json file.') + if len(root_folders) != 1: + raise InvalidPluginError('Wrong plugin structure.') + + if plugin_folder is None or plugin_folder not in root_folders: + raise InvalidPluginError('Missing plugin.json file.') archive.extractall(self.plugin_dir) return os.path.join(self.plugin_dir, plugin_folder) def install_plugin(self, plugin): - file = self.__get_plugin_file(plugin) + file = self._get_plugin_file(plugin) if not os.path.isdir(self.plugin_dir): os.makedirs(self.plugin_dir) - plugin_path = self.__check_plugin_archive(file) + plugin_path = self._check_plugin_archive(file) try: - plugin_class = self.__source_plugin(plugin_path) + plugin_class = self._source_plugin(plugin_path) plugin_instance = plugin_class(self, {}) - plugin_instance.install() + plugin_installed = plugin_instance.install() except InvalidPluginError as e: shutil.rmtree(plugin_path) raise e @@ -193,8 +200,10 @@ def install_plugin(self, plugin): if os.path.isdir(self.__plugin_download_dir): shutil.rmtree(self.__plugin_download_dir) + return plugin_installed + def uninstall_plugin(self, name): - self.__plugin_exists(name) + self._plugin_exists(name) # After the uninstall method was successful remove the plugin if self.plugin_list[name].uninstall(): @@ -203,9 +212,9 @@ def uninstall_plugin(self, name): return False def update_plugin(self, plugin_id): - self.__plugin_exists(plugin_id) + self._plugin_exists(plugin_id) return self.plugin_list[plugin_id].update() def configure_plugin(self, plugin_id): - self.__plugin_exists(plugin_id) + self._plugin_exists(plugin_id) return self.plugin_list[plugin_id].configure() diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 7ef492a561e..c3edc3cb6aa 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -578,6 +578,7 @@ def test_project_up_networks(self): 'bar': {'driver': None}, 'baz': {}, }, + plugins=None, ) project = Project.from_config( @@ -635,6 +636,7 @@ def test_up_with_ipam_config(self): }, }, }, + plugins=None, ) project = Project.from_config( @@ -697,7 +699,8 @@ def test_up_with_network_static_addresses(self): ] } } - } + }, + plugins=None ) project = Project.from_config( client=self.client, @@ -745,6 +748,7 @@ def test_up_with_network_static_addresses_missing_subnet(self): }, }, }, + plugins=None, ) project = Project.from_config( @@ -756,6 +760,34 @@ def test_up_with_network_static_addresses_missing_subnet(self): with self.assertRaises(ProjectError): project.up() + @v2_only() + def test_project_up_with_network_internal(self): + self.require_api_version('1.23') + config_data = config.Config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'networks': {'internal': None}, + }], + volumes={}, + networks={ + 'internal': {'driver': 'bridge', 'internal': True}, + }, + plugins=None, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + + network = self.client.networks(names=['composetest_internal'])[0] + + assert network['Internal'] is True + @v2_only() def test_project_up_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) @@ -769,6 +801,7 @@ def test_project_up_volumes(self): }], volumes={vol_name: {'driver': 'local'}}, networks={}, + plugins=None, ) project = Project.from_config( @@ -847,6 +880,7 @@ def test_initialize_volumes(self): }], volumes={vol_name: {}}, networks={}, + plugins=None, ) project = Project.from_config( @@ -872,6 +906,7 @@ def test_project_up_implicit_volume_driver(self): }], volumes={vol_name: {}}, networks={}, + plugins=None, ) project = Project.from_config( @@ -897,6 +932,7 @@ def test_initialize_volumes_invalid_volume_driver(self): }], volumes={vol_name: {'driver': 'foobar'}}, networks={}, + plugins=None, ) project = Project.from_config( @@ -920,6 +956,7 @@ def test_initialize_volumes_updated_driver(self): }], volumes={vol_name: {'driver': 'local'}}, networks={}, + plugins=None, ) project = Project.from_config( name='composetest', @@ -959,6 +996,7 @@ def test_initialize_volumes_updated_blank_driver(self): }], volumes={vol_name: {'driver': 'local'}}, networks={}, + plugins=None, ) project = Project.from_config( name='composetest', @@ -1000,6 +1038,7 @@ def test_initialize_volumes_external_volumes(self): vol_name: {'external': True, 'external_name': vol_name} }, networks=None, + plugins=None, ) project = Project.from_config( name='composetest', @@ -1025,6 +1064,7 @@ def test_initialize_volumes_inexistent_external_volume(self): vol_name: {'external': True, 'external_name': vol_name} }, networks=None, + plugins=None, ) project = Project.from_config( name='composetest', diff --git a/tests/unit/bundle_test.py b/tests/unit/bundle_test.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 2c90b29b72c..c312cd7ebd1 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -20,6 +20,7 @@ from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand from compose.const import IS_WINDOWS_PLATFORM +from compose.plugin_manager import PluginManager from compose.project import Project @@ -105,7 +106,8 @@ def test_run_interactive_passes_logs_false(self, mock_pseudo_terminal, mock_run_ 'service': {'image': 'busybox'} }), ) - command = TopLevelCommand(project) + plugin_manager = PluginManager('') + command = TopLevelCommand(project, plugin_manager) with pytest.raises(SystemExit): command.run({ @@ -140,8 +142,8 @@ def test_run_service_with_restart_always(self): } }), ) - - command = TopLevelCommand(project) + plugin_manager = PluginManager('') + command = TopLevelCommand(project, plugin_manager) command.run({ 'SERVICE': 'service', 'COMMAND': None, @@ -162,8 +164,8 @@ def test_run_service_with_restart_always(self): mock_client.create_host_config.call_args[1]['restart_policy']['Name'], 'always' ) - - command = TopLevelCommand(project) + plugin_manager = PluginManager('') + command = TopLevelCommand(project, plugin_manager) command.run({ 'SERVICE': 'service', 'COMMAND': None, @@ -192,7 +194,8 @@ def test_command_manula_and_service_ports_together(self): 'service': {'image': 'busybox'}, }), ) - command = TopLevelCommand(project) + plugin_manager = PluginManager('') + command = TopLevelCommand(project, plugin_manager) with self.assertRaises(UserError): command.run({ diff --git a/tests/unit/plugin_manager_test.py b/tests/unit/plugin_manager_test.py index e69de29bb2d..f45259a82c9 100644 --- a/tests/unit/plugin_manager_test.py +++ b/tests/unit/plugin_manager_test.py @@ -0,0 +1,441 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from .. import mock +from .. import unittest +from compose.plugin import Plugin +from compose.plugin_manager import InvalidPluginError +from compose.plugin_manager import InvalidPluginFileTypeError +from compose.plugin_manager import NoneLoadedConfigError +from compose.plugin_manager import PluginDoesNotExistError +from compose.plugin_manager import PluginManager + + +class PluginManagerTest(unittest.TestCase): + def setUp(self): + pass + + def _get_helper_class(self, attributes=None): + class Foo(object): + def __init__(self): + if attributes is not None: + for (attribute_name, attribute_value) in attributes.items(): + setattr(self, attribute_name, attribute_value) + + return Foo() + + def _get_archive_mock(self, files): + class MockArchive: + def __init__(self, files): + self.files = files + + def __iter__(self): + return iter(self.files) + + def write(self, fname): + self.files.append(fname) + + def namelist(self): + return self.files + + def extractall(self, destination): + pass + + return MockArchive(files) + + @staticmethod + def _get_plugin_manager_with_plugin(): + plugin_manager = PluginManager('plugin_dir') + + plugin_copy = Plugin + plugin_copy.__init__ = lambda a, b, c: None + plugin_copy.path = 'path' + plugin_manager.plugin_list = { + 'plugin': plugin_copy(plugin_manager, {}) + } + + return plugin_manager + + def test_load_config(self): + plugin_manager = PluginManager('') + plugin_manager.load_config('', {}) + self.assertEquals(plugin_manager.config, False) + + def test_get_plugin_paths_invalid_dir(self): + with mock.patch('compose.plugin_manager.os.path.isdir') as mock_isdir: + mock_isdir.return_value = False + plugin_manager = PluginManager('plugin_dir') + self.assertEquals(plugin_manager._get_plugin_paths(), {}) + + def test_get_plugin_paths_valid_dir(self): + with mock.patch('compose.plugin_manager.os.path.isdir') as mock_isdir,\ + mock.patch('compose.plugin_manager.os.listdir') as mock_listdir: + plugin_manager = PluginManager('') + plugin_manager.plugin_dir = 'plugin_dir' + mock_isdir.side_effect = [True, True, False, True] + mock_listdir.return_value = ['plugin_1', 'plugin_2', 'plugin_3'] + + self.assertEquals( + plugin_manager._get_plugin_paths(), + { + 'plugin_1': 'plugin_dir/plugin_1', + 'plugin_3': 'plugin_dir/plugin_3' + } + ) + + def test_source_plugin_missing_init_file(self): + plugin_manager = PluginManager('') + + with mock.patch('compose.plugin_manager.os.path.isfile') as mock_isfile: + mock_isfile.return_value = False + + with self.assertRaises(InvalidPluginError) as e: + plugin_manager._source_plugin('plugin_path') + + self.assertEqual(str(e.exception), "Missing __init__.py file.") + + def test_source_plugin_missing_plugin_attribute(self): + plugin_manager = PluginManager('') + + with mock.patch('compose.plugin_manager.os.path.isfile') as mock_isfile,\ + mock.patch('imp.load_source') as mock_load_source, \ + mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [] + mock_load_source.return_value = self._get_helper_class() + mock_isfile.return_value = True + + with self.assertRaises(InvalidPluginError) as e: + plugin_manager._source_plugin('plugin') + + self.assertEqual( + str(e.exception), + "Plugin 'plugin' is not a plugin. Missing plugin attribute." + ) + + def test_source_plugin_invalid_plugin_attribute(self): + plugin_manager = PluginManager('') + + with mock.patch('compose.plugin_manager.os.path.isfile') as mock_isfile, \ + mock.patch('imp.load_source') as mock_load_source, \ + mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [] + mock_load_source.return_value = self._get_helper_class({ + 'plugin': self._get_helper_class().__class__ + }) + mock_isfile.return_value = True + + with self.assertRaises(InvalidPluginError) as e: + plugin_manager._source_plugin('plugin') + + self.assertEqual(str(e.exception), "Wrong plugin instance.") + + def test_source_plugin_valid_plugin(self): + plugin_manager = PluginManager('') + + with mock.patch('compose.plugin_manager.os.path.isfile') as mock_isfile, \ + mock.patch('imp.load_source') as mock_load_source, \ + mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [] + mock_load_source.return_value = self._get_helper_class({ + 'plugin': Plugin + }) + mock_isfile.return_value = True + + self.assertEqual(plugin_manager._source_plugin('plugin'), Plugin) + + def test_get_plugin_classes_invalid_dir(self): + plugin_manager = PluginManager('') + self.assertEquals(plugin_manager._get_plugin_classes(), {}) + + def test_get_plugin_classes_valid_plugins(self): + with mock.patch('compose.plugin_manager.os.path.isdir') as mock_isdir, \ + mock.patch('compose.plugin_manager.os.path.isfile') as mock_isfile, \ + mock.patch('compose.plugin_manager.os.listdir') as mock_listdir, \ + mock.patch('imp.load_source') as mock_load_source, \ + mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [] + plugin_manager = PluginManager('') + plugin_manager.plugin_classes = None + mock_load_source.return_value = self._get_helper_class({ + 'plugin': Plugin + }) + mock_isdir.return_value = True + mock_isfile.return_value = True + mock_listdir.return_value = ['plugin_1', 'plugin_2'] + + self.assertEquals(plugin_manager._get_plugin_classes(), { + 'plugin_1': Plugin, + 'plugin_2': Plugin + }) + + def test_load_plugins_none_loaded_config(self): + plugin_manager = PluginManager('') + + with self.assertRaises(NoneLoadedConfigError) as e: + plugin_manager._load_plugins() + + self.assertEquals( + str(e.exception), + "The configuration wasn't loaded for the plugin manager. " + "Plugins can only instantiated after that." + ) + + def test_load_plugins(self): + with mock.patch('compose.plugin_manager.os.path.isdir') as mock_isdir, \ + mock.patch('compose.plugin_manager.os.path.isfile') as mock_isfile, \ + mock.patch('compose.plugin_manager.os.listdir') as mock_listdir, \ + mock.patch('imp.load_source') as mock_load_source, \ + mock.patch.object(Plugin, '__init__') as mock_plugin, \ + mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [] + mock_plugin.return_value = None + + plugin_manager = PluginManager('') + plugin_manager.plugin_classes = None + + mock_load_source.return_value = self._get_helper_class({ + 'plugin': Plugin + }) + mock_isdir.return_value = True + mock_isfile.return_value = True + mock_listdir.return_value = ['plugin_1', 'plugin_2'] + plugin_manager.load_config('', {}) + + loaded_plugins = plugin_manager._load_plugins() + self.assertEquals('plugin_1' in loaded_plugins.keys(), True) + self.assertEquals('plugin_2' in loaded_plugins.keys(), True) + self.assertEquals(isinstance(loaded_plugins['plugin_1'], Plugin), True) + self.assertEquals(isinstance(loaded_plugins['plugin_2'], Plugin), True) + + """ + def test_get_plugins(self): + assert False + """ + + def get_loaded_plugin_manager(self): + with mock.patch('compose.plugin_manager.os.path.isdir') as mock_isdir, \ + mock.patch('compose.plugin_manager.os.path.isfile') as mock_isfile, \ + mock.patch('compose.plugin_manager.os.listdir') as mock_listdir, \ + mock.patch('imp.load_source') as mock_load_source, \ + mock.patch.object(Plugin, '__init__') as mock_plugin, \ + mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [] + mock_plugin.return_value = None + mock_load_source.return_value = self._get_helper_class({ + 'plugin': Plugin + }) + mock_isdir.return_value = True + mock_isfile.return_value = True + mock_listdir.return_value = ['plugin_1', 'plugin_2'] + + plugin_manager = PluginManager('plugin_folder') + plugin_manager.load_config('', {}) + return plugin_manager + + def test_plugin_exists(self): + plugin_manager = self.get_loaded_plugin_manager() + + with self.assertRaises(PluginDoesNotExistError) as e: + plugin_manager._plugin_exists('no_plugin') + + self.assertEqual(str(e.exception), "Plugin 'no_plugin' doesn't exists") + plugin_manager._plugin_exists('plugin_1') + + def test_is_plugin_installed(self): + plugin_manager = self.get_loaded_plugin_manager() + + self.assertEquals(plugin_manager.is_plugin_installed('plugin_1'), True) + self.assertEquals(plugin_manager.is_plugin_installed('no_plugin'), False) + + def test_get_plugin_file(self): + with mock.patch('compose.plugin_manager.request.urlretrieve') as mock_urlretrieve, \ + mock.patch('compose.plugin_manager.os.path.isfile') as mock_isfile, \ + mock.patch('compose.plugin_manager.os.path.realpath') as mock_realpath, \ + mock.patch('os.mkdir') as mock_mkdir: + plugin_manager = PluginManager('plugin_dir') + + self.assertEquals( + plugin_manager._get_plugin_file('plugin_name'), + 'plugin_dir/.downloads/plugin_name' + ) + + mock_urlretrieve.side_effect = ValueError() + mock_isfile.return_value = False + mock_mkdir.return_value = True + + with self.assertRaises(InvalidPluginError) as e: + plugin_manager._get_plugin_file('no_plugin') + + self.assertEqual(str(e.exception), "Invalid plugin url or file given.") + + mock_isfile.return_value = True + mock_realpath.return_value = '/real/path/to/plugin/plugin_name' + + self.assertEquals( + plugin_manager._get_plugin_file('plugin_name'), + '/real/path/to/plugin/plugin_name' + ) + + def test_check_plugin_archive(self): + with mock.patch('compose.plugin_manager.zipfile.is_zipfile') as mock_is_zipfile, \ + mock.patch('compose.plugin_manager.zipfile.ZipFile') as mock_zipfile, \ + mock.patch('compose.plugin_manager.tarfile.is_tarfile') as mock_is_tarfile, \ + mock.patch('compose.plugin_manager.tarfile.TarFile') as mock_tarfile: + plugin_manager = PluginManager('plugin_dir') + mock_is_zipfile.return_value = False + mock_is_tarfile.return_value = False + with self.assertRaises(InvalidPluginFileTypeError) as e: + plugin_manager._check_plugin_archive('no_plugin') + + self.assertEqual(str(e.exception), "Invalid file type.") + + mock_is_zipfile.return_value = True + mock_zipfile.return_value = self._get_archive_mock([]) + + with self.assertRaises(InvalidPluginError) as e: + plugin_manager._check_plugin_archive('no_plugin') + + self.assertEqual(str(e.exception), "Wrong plugin structure.") + + mock_zipfile.return_value = self._get_archive_mock([ + 'root_dir/', + 'second_root_dir/', + 'root_dir/plugin.json' + ]) + + with self.assertRaises(InvalidPluginError) as e: + plugin_manager._check_plugin_archive('plugin.zip') + + self.assertEqual(str(e.exception), "Wrong plugin structure.") + + mock_zipfile.return_value = self._get_archive_mock([ + 'root_dir/', + 'root_dir/plugin.js' + ]) + + with self.assertRaises(InvalidPluginError) as e: + plugin_manager._check_plugin_archive('plugin.zip') + + self.assertEqual(str(e.exception), "Missing plugin.json file.") + + mock_zipfile.return_value = self._get_archive_mock([ + 'root_dir/', + 'root_dir/plugin.json' + ]) + + result = plugin_manager._check_plugin_archive('plugin.zip') + self.assertEqual(result, "plugin_dir/root_dir") + + mock_is_zipfile.return_value = False + mock_is_tarfile.return_value = True + mock_tarfile.return_value = self._get_archive_mock([ + 'root_dir/', + 'root_dir/plugin.json' + ]) + + result = plugin_manager._check_plugin_archive('plugin.zip') + self.assertEqual(result, "plugin_dir/root_dir") + + def test_install_plugin(self): + with mock.patch('compose.plugin_manager.request.urlretrieve') as mock_urlretrieve, \ + mock.patch('compose.plugin_manager.os.path.isfile') as mock_isfile, \ + mock.patch('compose.plugin_manager.os.path.realpath') as mock_realpath, \ + mock.patch('compose.plugin_manager.os.path.isdir') as mock_isdir, \ + mock.patch('compose.plugin_manager.os.makedirs') as mock_makedirs, \ + mock.patch('compose.plugin_manager.zipfile.is_zipfile') as mock_is_zipfile, \ + mock.patch('compose.plugin_manager.tarfile.is_tarfile') as mock_is_tarfile, \ + mock.patch('compose.plugin_manager.zipfile.ZipFile') as mock_zipfile, \ + mock.patch('imp.load_source') as mock_load_source, \ + mock.patch('shutil.rmtree') as mock_rmtree, \ + mock.patch('os.mkdir') as mock_mkdir, \ + mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [] + mock_urlretrieve.side_effect = ValueError() + mock_isfile.return_value = True + mock_mkdir.return_value = True + mock_realpath.return_value = '/real/path/to/plugin/plugin_name' + + mock_isdir.return_value = False + + mock_is_tarfile.return_value = False + mock_is_zipfile.return_value = True + mock_zipfile.return_value = self._get_archive_mock([ + 'root_dir/', + 'root_dir/plugin.json' + ]) + mock_load_source.return_value = None + self._get_helper_class({ + 'plugin': Plugin + }) + + plugin_manager = PluginManager('plugin_dir') + + with self.assertRaises(InvalidPluginError) as e: + plugin_manager.install_plugin('plugin') + self.assertTrue(mock_makedirs.called) + self.assertTrue(mock_rmtree.called) + + self.assertEqual( + str(e.exception), + "Plugin 'root_dir' is not a plugin. Missing plugin attribute." + ) + + plugin_copy = Plugin + plugin_copy.__init__ = lambda a, b, c: None + plugin_copy.install = lambda a: False + + mock_load_source.return_value = self._get_helper_class({ + 'plugin': plugin_copy + }) + + self.assertEquals(plugin_manager.install_plugin('plugin'), False) + + plugin_copy.install = lambda a: True + + mock_isdir.return_value = True + mock_load_source.return_value = self._get_helper_class({ + 'plugin': plugin_copy + }) + + self.assertEquals(plugin_manager.install_plugin('plugin'), True) + + def test_uninstall_plugin(self): + with mock.patch('shutil.rmtree') as mock_rmtree: + plugin_manager = self._get_plugin_manager_with_plugin() + + with self.assertRaises(PluginDoesNotExistError) as e: + plugin_manager.uninstall_plugin('none_plugin') + + self.assertEqual( + str(e.exception), + "Plugin 'none_plugin' doesn't exists" + ) + + self.assertTrue(plugin_manager.uninstall_plugin('plugin')) + self.assertTrue(mock_rmtree.called) + + def test_update_plugin(self): + plugin_manager = self._get_plugin_manager_with_plugin() + + with self.assertRaises(PluginDoesNotExistError) as e: + plugin_manager.update_plugin('none_plugin') + + self.assertEqual( + str(e.exception), + "Plugin 'none_plugin' doesn't exists" + ) + + self.assertEquals(plugin_manager.update_plugin('plugin'), None) + + def test_configure_plugin(self): + plugin_manager = self._get_plugin_manager_with_plugin() + + with self.assertRaises(PluginDoesNotExistError) as e: + plugin_manager.configure_plugin('none_plugin') + + self.assertEqual( + str(e.exception), + "Plugin 'none_plugin' doesn't exists" + ) + + self.assertEquals(plugin_manager.configure_plugin('plugin'), None) diff --git a/tests/unit/plugin_test.py b/tests/unit/plugin_test.py index b28632d5ff3..17fd87f4a3c 100644 --- a/tests/unit/plugin_test.py +++ b/tests/unit/plugin_test.py @@ -2,45 +2,188 @@ from __future__ import unicode_literals import json +import re import pytest +import tests from .. import mock from .. import unittest +from compose.cli.main import TopLevelCommand +from compose.plugin import compose_command +from compose.plugin import compose_patch from compose.plugin import Plugin +from compose.plugin import PluginCommandError from compose.plugin import PluginJsonFileError +class ClassToPatch(object): + def __init__(self): + self.name = "name" + + def get_name(self, prefix): + return prefix + self.name + + +def test_compose_patch_class(): + @compose_patch(ClassToPatch, "get_name") + def patched_get_name(self, original_fnc, prefix): + original_return = original_fnc(self, prefix) + return original_return + prefix[::-1] + + mock_instance = ClassToPatch() + assert mock_instance.get_name("abc|") == "abc|name|cba" + assert hasattr(mock_instance.get_name, '__standalone__') is False + + +def fnc_to_patch(string): + return string + +fnc_to_patch.__standalone__ = True + + +def test_compose_patch_function(): + @compose_patch(tests.unit.plugin_test, "fnc_to_patch") + def patch_text_fnc(original_fnc, string): + return original_fnc(string) * 2 + + assert fnc_to_patch("|test|") == "|test||test|" + assert fnc_to_patch.__standalone__ is True + + +def test_compose_command(): + @compose_command() + def test_command(self): + """ + Test command + + Usage: config [options] + + Options: + -o, --option Option + """ + return True + + @compose_command(standalone=True) + def stest_command(self): + """ + Second test command + + Usage: config [options] + + Options: + -o, --option Option + """ + return False + + if hasattr(TopLevelCommand, '__modified_doc__'): + assert re.search( + r'test_command\s+Test command', + TopLevelCommand.__modified_doc__ + ) is not None + assert re.search( + r'stest_command\s+Second test command', + TopLevelCommand.__modified_doc__ + ) is not None + else: + assert re.search( + r'test_command\s+Test command', + TopLevelCommand.__doc__ + ) is not None + assert re.search( + r'stest_command\s+Second test command', + TopLevelCommand.__doc__ + ) is not None + + top_level_command = TopLevelCommand(None, None, None) + + assert TopLevelCommand.test_command.__standalone__ is False + assert TopLevelCommand.stest_command.__standalone__ is True + + assert top_level_command.test_command() is True + assert top_level_command.stest_command() is False + + with pytest.raises(PluginCommandError): + test_command(None) + + with pytest.raises(PluginCommandError): + stest_command(None) + + class PluginTest(unittest.TestCase): valid_plugin_info = {'name': 'Plugin name', 'version': '1.0.0', 'description': 'Plugin description'} invalid_plugin_info = {'var1': 'value1'} def setUp(self): - pass + self.json_file = json.dumps(self.valid_plugin_info, False, True) - @mock.patch('compose.plugin.os.path.isfile') - def test_load_plugin_info_from_file(self, mock_isfile): - mock_isfile.return_value = True - json_file = json.dumps(self.valid_plugin_info, False, True) - m = mock.mock_open(read_data=json_file) + def getPluginFileMock(self): + return mock.mock_open(read_data=self.json_file) - with mock.patch('compose.plugin.open', m, create=True): - Plugin.__init__ = mock.Mock(return_value=None) - plugin = Plugin(None) # TODO plugin_manager mock - plugin.load_plugin_info_from_file('plugin.json') + def _get_loaded_plugin(self): + with mock.patch('compose.plugin.os.path.abspath') as mock_abspath, \ + mock.patch('compose.plugin.os.path.isfile') as mock_isfile, \ + mock.patch('compose.plugin.open', self.getPluginFileMock(), create=True): + mock_abspath.return_value = '/plugins/plugin_id' + mock_isfile.return_value = True + + return Plugin(None, {}) + + def test_init(self): + with mock.patch('compose.plugin.os.path.abspath') as mock_abspath, \ + mock.patch('compose.plugin.os.path.isfile') as mock_isfile, \ + mock.patch('compose.plugin.open', self.getPluginFileMock(), create=True): + mock_abspath.return_value = '/plugins/plugin_id' + mock_isfile.return_value = True - assert plugin.name == 'Plugin name' - assert plugin.version == '1.0.0' - assert plugin.description == 'Plugin description' + self.assertIsInstance(Plugin(None, {}), Plugin) def test_check_required_plugin_file_settings_error(self): - with pytest.raises(PluginJsonFileError): + with self.assertRaises(PluginJsonFileError) as e: Plugin.check_required_plugin_file_settings(self.invalid_plugin_info, Plugin.required_fields) + self.assertEqual(str(e.exception), "Missing json attribute 'name'") + def test_check_required_plugin_file_settings_success(self): check = Plugin.check_required_plugin_file_settings( self.valid_plugin_info, Plugin.required_fields ) - assert check is True + self.assertEquals(check, True) + + def test_load_plugin_info_from_file_not_found(self): + plugin = self._get_loaded_plugin() + + with self.assertRaises(PluginJsonFileError) as e: + plugin.load_plugin_info_from_file('invalid.json') + + self.assertEqual(str(e.exception), "JSON plugin file not found") + + def test_load_plugin_info_from_file(self): + plugin = self._get_loaded_plugin() + + with mock.patch('compose.plugin.os.path.isfile') as mock_isfile, \ + mock.patch('compose.plugin.open', self.getPluginFileMock(), create=True): + mock_isfile.return_value = True + plugin.load_plugin_info_from_file('plugin.json') + + self.assertEquals(plugin.name, 'Plugin name') + self.assertEquals(plugin.version, '1.0.0') + self.assertEquals(plugin.description, 'Plugin description') + + def test_install(self): + plugin = self._get_loaded_plugin() + self.assertEquals(plugin.install(), True) + + def test_uninstall(self): + plugin = self._get_loaded_plugin() + self.assertEquals(plugin.uninstall(), True) + + def test_update(self): + plugin = self._get_loaded_plugin() + self.assertEquals(plugin.update(), None) + + def test_configure(self): + plugin = self._get_loaded_plugin() + self.assertEquals(plugin.configure(), None) From 2241bff96c6e20d0ab3cb73cc5a8709035a2f7cc Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Wed, 31 Aug 2016 00:17:16 +0200 Subject: [PATCH 08/14] Small optimization related to the plugin dir creation Signed-off-by: Alexander Schneider --- compose/plugin_manager.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/compose/plugin_manager.py b/compose/plugin_manager.py index efe2bef37f5..05897e39b55 100644 --- a/compose/plugin_manager.py +++ b/compose/plugin_manager.py @@ -142,7 +142,7 @@ def is_plugin_installed(self, plugin_id): def _get_plugin_file(self, plugin): try: if not os.path.isdir(self.__plugin_download_dir): - os.mkdir(self.__plugin_download_dir) + os.makedirs(self.__plugin_download_dir) file = os.path.join(self.__plugin_download_dir, os.path.basename(plugin)) request.urlretrieve(plugin, file) @@ -182,11 +182,10 @@ def _check_plugin_archive(self, file): return os.path.join(self.plugin_dir, plugin_folder) def install_plugin(self, plugin): - file = self._get_plugin_file(plugin) - if not os.path.isdir(self.plugin_dir): os.makedirs(self.plugin_dir) + file = self._get_plugin_file(plugin) plugin_path = self._check_plugin_archive(file) try: From 9d95ac0787924817cc8254fc49ef17a16e028949 Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Wed, 31 Aug 2016 22:59:24 +0200 Subject: [PATCH 09/14] Adjust acceptance tests Signed-off-by: Alexander Schneider --- tests/acceptance/cli_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a02d0e99eb5..7ded474d2e4 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -180,6 +180,7 @@ def test_config_default(self): 'version': '2.0', 'volumes': {'data': {'driver': 'local'}}, 'networks': {'front': {}}, + 'plugins': {}, 'services': { 'web': { 'build': { @@ -221,6 +222,7 @@ def test_config_restart(self): }, }, 'networks': {}, + 'plugins': {}, 'volumes': {}, } @@ -246,6 +248,7 @@ def test_config_v1(self): }, }, 'networks': {}, + 'plugins': {}, 'volumes': {}, } From a7abcd2b58a85ae096e81bb0ce06e3a76ecbbc89 Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Thu, 1 Sep 2016 01:14:38 +0200 Subject: [PATCH 10/14] Ignore .download folder at indexing plugins Signed-off-by: Alexander Schneider --- compose/plugin_manager.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compose/plugin_manager.py b/compose/plugin_manager.py index 05897e39b55..ca25e81a1f9 100644 --- a/compose/plugin_manager.py +++ b/compose/plugin_manager.py @@ -65,10 +65,11 @@ def _get_plugin_paths(self): if os.path.isdir(self.plugin_dir): for current_plugin_dir in os.listdir(self.plugin_dir): - plugin_path = os.path.join(self.plugin_dir, current_plugin_dir) + if current_plugin_dir != '.downloads': + plugin_path = os.path.join(self.plugin_dir, current_plugin_dir) - if os.path.isdir(plugin_path): - paths[current_plugin_dir] = plugin_path + if os.path.isdir(plugin_path): + paths[current_plugin_dir] = plugin_path return paths From dc453350f007319e45c1bf60200f1f491a49522a Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Fri, 2 Sep 2016 20:03:57 +0200 Subject: [PATCH 11/14] Check for required plugins defined at config file and for plugin version Add old version for plugin update method Signed-off-by: Alexander Schneider --- compose/config/config_schema_v2.0.json | 12 +++-- compose/plugin.py | 2 +- compose/plugin_manager.py | 41 ++++++++++++++++- tests/unit/plugin_manager_test.py | 62 +++++++++++++++++++++++++- tests/unit/plugin_test.py | 2 +- 5 files changed, 109 insertions(+), 10 deletions(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 83c83895c24..20b53324fbf 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -327,12 +327,16 @@ }, "plugin": { - "version": { - "type": "string" - }, "id": "#/definitions/plugin", "type": "object", - "options": {"$ref": "#/definitions/list_or_dict"} + "properties": { + "version": { + "type": "string" + }, + "options": { + "$ref": "#/definitions/list_or_dict" + } + } } } } diff --git a/compose/plugin.py b/compose/plugin.py index c47bc5f686b..ac8b860ac3e 100644 --- a/compose/plugin.py +++ b/compose/plugin.py @@ -151,7 +151,7 @@ def install(self): def uninstall(self): return True - def update(self): + def update(self, old_version): return None def configure(self): diff --git a/compose/plugin_manager.py b/compose/plugin_manager.py index ca25e81a1f9..ce19b9c5895 100644 --- a/compose/plugin_manager.py +++ b/compose/plugin_manager.py @@ -6,6 +6,7 @@ import shutil import sys import tarfile +from distutils.version import LooseVersion from compose.cli.command import get_config_path_from_options from compose.config import config @@ -39,6 +40,10 @@ class NoneLoadedConfigError(PluginError): pass +class PluginRequirementsError(PluginError): + pass + + class PluginManager(object): def __init__(self, plugin_dir): self.plugin_dir = plugin_dir @@ -110,6 +115,26 @@ def _get_plugin_classes(self): return self.plugin_classes + @staticmethod + def _check_required_plugins(plugins_config, plugins): + plugin_diff = list(set(plugins_config.keys()) - set(plugins.keys())) + + if len(plugin_diff) > 0: + plugin_diff = sorted(plugin_diff) + raise PluginRequirementsError( + "Missing required plugins: '{}'".format("', '".join(plugin_diff)) + ) + + for (plugin_id, plugin_config) in plugins_config.items(): + if 'version' in plugin_config and \ + LooseVersion(plugins[plugin_id].version) < LooseVersion(plugin_config['version']): + raise PluginRequirementsError( + "Plugin '{}' must at least version '{}'".format( + plugin_id, + LooseVersion(plugin_config['version']) + ) + ) + def _load_plugins(self): if self.plugin_list is None: if self.config is None: @@ -120,10 +145,19 @@ def _load_plugins(self): self.plugin_list = {} for (plugin_id, plugin_class) in self._get_plugin_classes().items(): - plugin_config = plugins_config[plugin_id] if plugin_id in plugins_config else {} + if plugin_id in plugins_config and 'options' in plugins_config[plugin_id]: + plugin_config = plugins_config[plugin_id].options + else: + plugin_config = {} + plugin_instance = plugin_class(self, plugin_config) self.plugin_list[plugin_id] = plugin_instance + self._check_required_plugins( + plugins_config, + self.plugin_list + ) + return self.plugin_list def get_plugins(self): @@ -213,7 +247,10 @@ def uninstall_plugin(self, name): def update_plugin(self, plugin_id): self._plugin_exists(plugin_id) - return self.plugin_list[plugin_id].update() + plugins = self.get_plugins() + plugin_version = plugins[plugin_id].version + + return self.plugin_list[plugin_id].update(plugin_version) def configure_plugin(self, plugin_id): self._plugin_exists(plugin_id) diff --git a/tests/unit/plugin_manager_test.py b/tests/unit/plugin_manager_test.py index f45259a82c9..a8c8736ded1 100644 --- a/tests/unit/plugin_manager_test.py +++ b/tests/unit/plugin_manager_test.py @@ -9,6 +9,7 @@ from compose.plugin_manager import NoneLoadedConfigError from compose.plugin_manager import PluginDoesNotExistError from compose.plugin_manager import PluginManager +from compose.plugin_manager import PluginRequirementsError class PluginManagerTest(unittest.TestCase): @@ -26,8 +27,8 @@ def __init__(self): def _get_archive_mock(self, files): class MockArchive: - def __init__(self, files): - self.files = files + def __init__(self, archive_files): + self.files = archive_files def __iter__(self): return iter(self.files) @@ -50,6 +51,7 @@ def _get_plugin_manager_with_plugin(): plugin_copy = Plugin plugin_copy.__init__ = lambda a, b, c: None plugin_copy.path = 'path' + plugin_copy.version = '0.0.1' plugin_manager.plugin_list = { 'plugin': plugin_copy(plugin_manager, {}) } @@ -168,6 +170,62 @@ def test_get_plugin_classes_valid_plugins(self): 'plugin_2': Plugin }) + def test_check_required_plugins(self): + with mock.patch.object(Plugin, '__init__') as mock_plugin: + mock_plugin.return_value = None + plugin_manager = PluginManager('') + + plugins = {} + plugins_config = { + 'plugin_1': {}, + 'plugin_2': {} + } + + with self.assertRaises(PluginRequirementsError) as e: + plugin_manager._check_required_plugins(plugins_config, plugins) + + self.assertEqual( + str(e.exception), + "Missing required plugins: 'plugin_1', 'plugin_2'" + ) + + plugin_1 = Plugin(plugin_manager, {}) + plugin_1.version = '0.0.1' + + plugin_2 = Plugin(plugin_manager, {}) + plugin_2.version = '2.0.1' + + plugins = { + 'plugin_1': plugin_1, + 'plugin_2': plugin_2 + } + plugins_config = { + 'plugin_1': { + 'version': '2.0.0' + }, + 'plugin_2': {} + } + + with self.assertRaises(PluginRequirementsError) as e: + plugin_manager._check_required_plugins(plugins_config, plugins) + + self.assertEqual( + str(e.exception), + "Plugin 'plugin_1' must at least version '2.0.0'" + ) + + plugins_config = { + 'plugin_1': { + 'version': '0.0.1' + }, + 'plugin_2': { + 'version': '1.0.0' + } + } + + result = plugin_manager._check_required_plugins(plugins_config, plugins) + self.assertEquals(result, None) + def test_load_plugins_none_loaded_config(self): plugin_manager = PluginManager('') diff --git a/tests/unit/plugin_test.py b/tests/unit/plugin_test.py index 17fd87f4a3c..2293b11678f 100644 --- a/tests/unit/plugin_test.py +++ b/tests/unit/plugin_test.py @@ -182,7 +182,7 @@ def test_uninstall(self): def test_update(self): plugin = self._get_loaded_plugin() - self.assertEquals(plugin.update(), None) + self.assertEquals(plugin.update('0.0.1'), None) def test_configure(self): plugin = self._get_loaded_plugin() From c204ed8fcd91f2ae5c40a43c03ed9455033260d6 Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Fri, 2 Sep 2016 22:04:24 +0200 Subject: [PATCH 12/14] Adjust paths for unit tests Signed-off-by: Alexander Schneider --- tests/unit/plugin_manager_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/unit/plugin_manager_test.py b/tests/unit/plugin_manager_test.py index a8c8736ded1..4792ef611e1 100644 --- a/tests/unit/plugin_manager_test.py +++ b/tests/unit/plugin_manager_test.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +import os + from .. import mock from .. import unittest from compose.plugin import Plugin @@ -80,8 +82,8 @@ def test_get_plugin_paths_valid_dir(self): self.assertEquals( plugin_manager._get_plugin_paths(), { - 'plugin_1': 'plugin_dir/plugin_1', - 'plugin_3': 'plugin_dir/plugin_3' + 'plugin_1': os.path.join('plugin_dir', 'plugin_1'), + 'plugin_3': os.path.join('plugin_dir', 'plugin_3') } ) @@ -314,7 +316,7 @@ def test_get_plugin_file(self): self.assertEquals( plugin_manager._get_plugin_file('plugin_name'), - 'plugin_dir/.downloads/plugin_name' + os.path.join('plugin_dir', '.downloads', 'plugin_name') ) mock_urlretrieve.side_effect = ValueError() @@ -392,7 +394,7 @@ def test_check_plugin_archive(self): ]) result = plugin_manager._check_plugin_archive('plugin.zip') - self.assertEqual(result, "plugin_dir/root_dir") + self.assertEqual(result, os.path.join('plugin_dir', 'root_dir')) def test_install_plugin(self): with mock.patch('compose.plugin_manager.request.urlretrieve') as mock_urlretrieve, \ From 9dc68bc95f51c525ec6d6c5a9340f7a2431b02ec Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Fri, 2 Sep 2016 22:10:23 +0200 Subject: [PATCH 13/14] Fix a path again for unit tests Signed-off-by: Alexander Schneider --- tests/unit/plugin_manager_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/plugin_manager_test.py b/tests/unit/plugin_manager_test.py index 4792ef611e1..f39f27e2147 100644 --- a/tests/unit/plugin_manager_test.py +++ b/tests/unit/plugin_manager_test.py @@ -384,7 +384,7 @@ def test_check_plugin_archive(self): ]) result = plugin_manager._check_plugin_archive('plugin.zip') - self.assertEqual(result, "plugin_dir/root_dir") + self.assertEqual(result, os.path.join('plugin_dir', 'root_dir')) mock_is_zipfile.return_value = False mock_is_tarfile.return_value = True From 560b69e153842aa030aa1012e58cfd04480aedf9 Mon Sep 17 00:00:00 2001 From: Alexander Schneider Date: Fri, 23 Sep 2016 11:15:59 +0200 Subject: [PATCH 14/14] Add plugin entries to config schema for v2.1 Adjust config v2.1 test Signed-off-by: Alexander Schneider --- compose/config/config_schema_v2.1.json | 24 ++++++++++++++++++++++++ tests/integration/project_test.py | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index de4ddf2509b..b530f248cd9 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -38,6 +38,17 @@ } }, "additionalProperties": false + }, + + "plugins": { + "id": "#/properties/plugins", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/plugin" + } + }, + "additionalProperties": false } }, @@ -314,6 +325,19 @@ } } } + }, + + "plugin": { + "id": "#/definitions/plugin", + "type": "object", + "properties": { + "version": { + "type": "string" + }, + "options": { + "$ref": "#/definitions/list_or_dict" + } + } } } } diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 3b98fc5145c..b2d49433a60 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -778,7 +778,8 @@ def test_up_with_network_link_local_ips(self): volumes={}, networks={ 'linklocaltest': {'driver': 'bridge'} - } + }, + plugins=None, ) project = Project.from_config( client=self.client,