From a6b016f83c33f80114dd4d939671344ed7674d4b Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Thu, 14 Mar 2019 20:45:33 +0100 Subject: [PATCH] Refactoring after getting feedback. --- bin/bundle-extensions | 58 ++++++++++-------- redash/__init__.py | 2 +- redash/extensions.py | 139 ++++++++++++++++++++++++++++++++++-------- redash/worker.py | 56 +++-------------- 4 files changed, 153 insertions(+), 102 deletions(-) diff --git a/bin/bundle-extensions b/bin/bundle-extensions index a920f03558..680804c462 100755 --- a/bin/bundle-extensions +++ b/bin/bundle-extensions @@ -1,35 +1,43 @@ #!/usr/bin/env python +"""Copy bundle extension files to the client/app/extension directory""" import os -from distutils.dir_util import copy_tree - -import importlib_metadata -import importlib_resources - -from redash.extensions import BUNDLE_DIRECTORY, resource_isdir, entry_point_module +from pathlib2 import Path +from shutil import copy +from redash import create_app, extensions # Make a directory for extensions and set it as an environment variable # to be picked up by webpack. -EXTENSIONS_RELATIVE_PATH = os.path.join('client', 'app', 'extensions') -EXTENSIONS_DIRECTORY = os.path.join( - os.path.dirname(os.path.dirname(__file__)), - EXTENSIONS_RELATIVE_PATH) - -if not os.path.exists(EXTENSIONS_DIRECTORY): - os.makedirs(EXTENSIONS_DIRECTORY) -os.environ["EXTENSIONS_DIRECTORY"] = EXTENSIONS_RELATIVE_PATH - - -for entry_point in importlib_metadata.entry_points().get('redash.extensions', []): - # Check if there is a "bundle" subdirectory/-package in the - # entrypoint's module and ignoring the entrypoint if not. - module = entry_point_module(entry_point) - if not resource_isdir(module, BUNDLE_DIRECTORY): +extensions_relative_path = Path('client', 'app', 'extensions') +extensions_directory = Path(__file__).parent.parent / extensions_relative_path + +if not extensions_directory.exists(): + extensions_directory.mkdir() +os.environ["EXTENSIONS_DIRECTORY"] = str(extensions_relative_path) + +# We need the app here for logging and to make sure the bundles have loaded. +app = create_app() + +bundles = extensions.bundles.items() +if bundles: + app.logger.info('Number of extension bundles found: %s', len(bundles)) +else: + app.logger.info('No extension bundles found.') + +for bundle_name, paths in extensions.bundles.items(): + # Shortcut in case not paths were found for the bundle + if not paths: + app.logger.info('No paths found for bundle "%s".', bundle_name) continue # The destination for the bundle files with the entry point name as the subdirectory - destination = os.path.join(EXTENSIONS_DIRECTORY, entry_point.name) + destination = Path(extensions_directory, bundle_name) + if not destination.exists(): + destination.mkdir() + # Copy the bundle directory from the module to its destination. - with importlib_resources.path(module, BUNDLE_DIRECTORY) as bundle_dir: - print("Copying {} to {}".format(bundle_dir, destination)) - copy_tree(str(bundle_dir), destination) + app.logger.info('Copying "%s" bundle to %s:', bundle_name, destination.resolve()) + for src_path in paths: + dest_path = destination / src_path.name + app.logger.info(" - {} -> {}".format(src_path, dest_path)) + copy(str(src_path), str(dest_path)) diff --git a/redash/__init__.py b/redash/__init__.py index 790549de63..51b209d2b6 100644 --- a/redash/__init__.py +++ b/redash/__init__.py @@ -131,7 +131,7 @@ def create_app(): limiter.init_app(app) handlers.init_app(app) configure_webpack(app) - extensions.init_extensions(app) + extensions.init_app(app) chrome_logger.init_app(app) users.init_app(app) diff --git a/redash/extensions.py b/redash/extensions.py index 4f4bd4f436..cb370f23ba 100644 --- a/redash/extensions.py +++ b/redash/extensions.py @@ -1,11 +1,22 @@ -from types import ModuleType +# -*- coding: utf-8 -*- +from collections import OrderedDict as odict + from importlib_resources import contents, is_resource, path from importlib_metadata import entry_points BUNDLE_DIRECTORY = 'bundle' # The global Redash extension registry -extensions = {} +extensions = odict() + +# The global Redash bundle registry +bundles = odict() + +# The periodic Celery tasks as provided by Redash extensions. +# This is separate from the internal periodic Celery tasks in +# celery_schedule since the extension task discovery phase is +# after the configuration has already happened. +periodic_tasks = odict() def resource_isdir(module, resource): @@ -25,16 +36,7 @@ def entry_point_module(entry_point): return entry_point.pattern.match(entry_point.value).group('module') -def module_bundle_files(module): - if not resource_isdir(module, BUNDLE_DIRECTORY): - return - - with path(module, BUNDLE_DIRECTORY) as bundle_dir: - # Copy content of extension bundle into extensions directory - return list(bundle_dir.rglob("*")) - - -def init_extensions(app): +def load_extensions(app): """Load the Redash extensions for the given Redash Flask app. The extension entry point can return any type of value but @@ -48,25 +50,108 @@ def extension(app): """ for entry_point in entry_points().get('redash.extensions', []): - app.logger.info('Loading Redash extension %s.', entry_point.name) - module = entry_point_module(entry_point) - # First of all, try to get a list of bundle files - extensions[entry_point.name]['resource_list'] = module_bundle_files(module) - + app.logger.info('Loading Redash extension "%s".', entry_point.name) try: # Then try to load the entry point (import and getattr) obj = entry_point.load() except (ImportError, AttributeError): # or move on - app.logger.error('Extension %s could not be found.', entry_point.name) - extensions[entry_point.name]['extension'] = None + app.logger.error('Redash extension "%s" could not be found.', entry_point.name) + continue + + if not callable(obj): + app.logger.error('Redash extension "%s" is not a callable.', entry_point.name) + continue + + # then simply call the loaded entry point. + extensions[entry_point.name] = obj(app) + + +def load_bundles(app): + """"Load bundles as defined in Redash extensions. + + The bundle entry point can be defined as a dotted path to a module + or a callable, but it won't be called but just used as a means + to find the files under its file system path. + + The name of the directory it looks for files in is "bundle". + + So a Python package with an extension bundle could look like this:: + + my_extensions/ + ├── __init__.py + └── wide_footer + ├── __init__.py + └── bundle + ├── extension.js + └── styles.css + + and would then need to register the bundle with an entry point + under the "redash.periodic_tasks" group, e.g. in your setup.py:: + + setup( + # ... + entry_points={ + "redash.bundles": [ + "wide_footer = my_extensions.wide_footer", + ] + # ... + }, + # ... + ) + + """ + for entry_point in entry_points().get('redash.bundles', []): + app.logger.info('Loading Redash bundle "%s".', entry_point.name) + module = entry_point_module(entry_point) + # Try to get a list of bundle files + if not resource_isdir(module, BUNDLE_DIRECTORY): + app.logger.error('Redash bundle directory "%s" could not be found.', entry_point.name) continue + with path(module, BUNDLE_DIRECTORY) as bundle_dir: + bundles[entry_point.name] = list(bundle_dir.rglob("*")) + + +def load_periodic_tasks(logger): + """Load the periodic tasks as defined in Redash extensions. + + The periodic task entry point needs to return a set of parameters + that can be passed to Celery's add_periodic_task: + + https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#entries + + E.g.:: + + def add_two_and_two(): + return { + 'name': 'add 2 and 2 every 10 seconds' + 'sig': add.s(2, 2), + 'schedule': 10.0, # in seconds or a timedelta + } + + and then registered with an entry point under the "redash.periodic_tasks" + group, e.g. in your setup.py:: + + setup( + # ... + entry_points={ + "redash.periodic_tasks": [ + "add_two_and_two = calculus.addition:add_two_and_two", + ] + # ... + }, + # ... + ) + """ + for entry_point in entry_points().get('redash.periodic_tasks', []): + logger.info('Loading periodic Redash tasks "%s" from "%s".', entry_point.name, entry_point.value) + try: + periodic_tasks[entry_point.name] = entry_point.load() + except (ImportError, AttributeError): + # and move on if it couldn't load it + logger.error('Periodic Redash task "%s" could not be found at "%s".', entry_point.name, entry_point.value) + - # Otherwise check if the loaded entry point is a module - if isinstance(obj, ModuleType): - app.logger.info('Extension %s is a module.', entry_point.name) - extensions[entry_point.name]['extension'] = obj - # or simply call the loaded entry point instead. - else: - app.logger.info('Extension %s is a callable.', entry_point.name) - extensions[entry_point.name]['extension'] = obj(app) +def init_app(app): + load_extensions(app) + load_bundles(app) diff --git a/redash/worker.py b/redash/worker.py index 328875602c..4f3b2e4b17 100644 --- a/redash/worker.py +++ b/redash/worker.py @@ -8,20 +8,13 @@ from celery.schedules import crontab from celery.signals import worker_process_init from celery.utils.log import get_logger -from importlib_metadata import entry_points -from redash import create_app, settings +from redash import create_app, extensions, settings from redash.metrics import celery as celery_metrics # noqa logger = get_logger(__name__) -# The periodic Celery tasks as provided by Redash extensions. -# This is separate from the internal periodic Celery tasks in -# celery_schedule since the extension task discovery phase is -# after the configuration has already happened. -extensions_schedule = {} - celery = Celery('redash', broker=settings.CELERY_BROKER, @@ -92,44 +85,9 @@ def init_celery_flask_app(**kwargs): @celery.on_after_configure.connect def add_periodic_tasks(sender, **kwargs): - """Load the periodic tasks as defined in Redash extensions. - - The periodic task entry point needs to return a set of parameters - that can be passed to Celery's add_periodic_task: - - https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#entries - - E.g.:: - - def add_two_and_two(): - return { - 'name': 'add 2 and 2 every 10 seconds' - 'sig': add.s(2, 2), - 'schedule': 10.0, # in seconds or a timedelta - } - - and then registered with an entry point under the "redash.periodic_tasks" - group, e.g. in your setup.py:: - - setup( - # ... - entry_points={ - "redash.periodic_tasks": [ - "add_two_and_two = calculus.addition:add_two_and_two", - ] - # ... - }, - # ... - ) - """ - for entry_point in entry_points().get('redash.periodic_tasks', []): - logger.info('Loading periodic Redash tasks %s from %s.', entry_point.name, entry_point.value) - try: - params = entry_point.load() - # Keep a record of our periodic tasks - extensions_schedule[entry_point.name] = params - # and add it to Celery's periodic task registry, too. - sender.add_periodic_task(**params) - except (ImportError, AttributeError): - # and move on if it couldn't load it - logger.error('Periodic Redash task %s could not be found at %s.', entry_point.name, entry_point.value) + """Load all periodic tasks from extensions and add them to Celery.""" + # Populate the redash.extensions.periodic_tasks dictionary + extensions.load_periodic_tasks(logger) + for params in extensions.periodic_tasks.values(): + # Add it to Celery's periodic task registry, too. + sender.add_periodic_task(**params)