Refactoring after getting feedback.

bin/bundle-extensions

@@ -1,35 +1,43 @@
 #!/usr/bin/env python
+"""Copy bundle extension files to the client/app/extension directory"""
 import os
-from distutils.dir_util import copy_tree
-
-import importlib_metadata
-import importlib_resources
-
-from redash.extensions import BUNDLE_DIRECTORY, resource_isdir, entry_point_module
+from pathlib2 import Path
+from shutil import copy
 
+from redash import create_app, extensions
 
 # Make a directory for extensions and set it as an environment variable
 # to be picked up by webpack.
-EXTENSIONS_RELATIVE_PATH = os.path.join('client', 'app', 'extensions')
-EXTENSIONS_DIRECTORY = os.path.join(
-    os.path.dirname(os.path.dirname(__file__)),
-    EXTENSIONS_RELATIVE_PATH)
-
-if not os.path.exists(EXTENSIONS_DIRECTORY):
-    os.makedirs(EXTENSIONS_DIRECTORY)
-os.environ["EXTENSIONS_DIRECTORY"] = EXTENSIONS_RELATIVE_PATH
-
-
-for entry_point in importlib_metadata.entry_points().get('redash.extensions', []):
-    # Check if there is a "bundle" subdirectory/-package in the
-    # entrypoint's module and ignoring the entrypoint if not.
-    module = entry_point_module(entry_point)
-    if not resource_isdir(module, BUNDLE_DIRECTORY):
+extensions_relative_path = Path('client', 'app', 'extensions')
+extensions_directory = Path(__file__).parent.parent / extensions_relative_path
+
+if not extensions_directory.exists():
+    extensions_directory.mkdir()
+os.environ["EXTENSIONS_DIRECTORY"] = str(extensions_relative_path)
+
+# We need the app here for logging and to make sure the bundles have loaded.
+app = create_app()
+
+bundles = extensions.bundles.items()
+if bundles:
+    app.logger.info('Number of extension bundles found: %s', len(bundles))
+else:
+    app.logger.info('No extension bundles found.')
+
+for bundle_name, paths in extensions.bundles.items():
+    # Shortcut in case not paths were found for the bundle
+    if not paths:
+        app.logger.info('No paths found for bundle "%s".', bundle_name)
         continue
 
     # The destination for the bundle files with the entry point name as the subdirectory
-    destination = os.path.join(EXTENSIONS_DIRECTORY, entry_point.name)
+    destination = Path(extensions_directory, bundle_name)
+    if not destination.exists():
+        destination.mkdir()
+
     # Copy the bundle directory from the module to its destination.
-    with importlib_resources.path(module, BUNDLE_DIRECTORY) as bundle_dir:
-        print("Copying {} to {}".format(bundle_dir, destination))
-        copy_tree(str(bundle_dir), destination)
+    app.logger.info('Copying "%s" bundle to %s:', bundle_name, destination.resolve())
+    for src_path in paths:
+        dest_path = destination / src_path.name
+        app.logger.info(" - {} -> {}".format(src_path, dest_path))
+        copy(str(src_path), str(dest_path))

redash/__init__.py

@@ -131,7 +131,7 @@ def create_app():
     limiter.init_app(app)
     handlers.init_app(app)
     configure_webpack(app)
-    extensions.init_extensions(app)
+    extensions.init_app(app)
     chrome_logger.init_app(app)
     users.init_app(app)
 

redash/extensions.py

@@ -1,11 +1,22 @@
-from types import ModuleType
+# -*- coding: utf-8 -*-
+from collections import OrderedDict as odict
+
 from importlib_resources import contents, is_resource, path
 from importlib_metadata import entry_points
 
 BUNDLE_DIRECTORY = 'bundle'
 
 # The global Redash extension registry
-extensions = {}
+extensions = odict()
+
+# The global Redash bundle registry
+bundles = odict()
+
+# The periodic Celery tasks as provided by Redash extensions.
+# This is separate from the internal periodic Celery tasks in
+# celery_schedule since the extension task discovery phase is
+# after the configuration has already happened.
+periodic_tasks = odict()
 
 
 def resource_isdir(module, resource):
@@ -25,16 +36,7 @@ def entry_point_module(entry_point):
     return entry_point.pattern.match(entry_point.value).group('module')
 
 
-def module_bundle_files(module):
-    if not resource_isdir(module, BUNDLE_DIRECTORY):
-        return
-
-    with path(module, BUNDLE_DIRECTORY) as bundle_dir:
-        # Copy content of extension bundle into extensions directory
-        return list(bundle_dir.rglob("*"))
-
-
-def init_extensions(app):
+def load_extensions(app):
     """Load the Redash extensions for the given Redash Flask app.
 
     The extension entry point can return any type of value but
@@ -48,25 +50,108 @@ def extension(app):
 
     """
     for entry_point in entry_points().get('redash.extensions', []):
-        app.logger.info('Loading Redash extension %s.', entry_point.name)
-        module = entry_point_module(entry_point)
-        # First of all, try to get a list of bundle files
-        extensions[entry_point.name]['resource_list'] = module_bundle_files(module)
-
+        app.logger.info('Loading Redash extension "%s".', entry_point.name)
         try:
             # Then try to load the entry point (import and getattr)
             obj = entry_point.load()
         except (ImportError, AttributeError):
             # or move on
-            app.logger.error('Extension %s could not be found.', entry_point.name)
-            extensions[entry_point.name]['extension'] = None
+            app.logger.error('Redash extension "%s" could not be found.', entry_point.name)
+            continue
+
+        if not callable(obj):
+            app.logger.error('Redash extension "%s" is not a callable.', entry_point.name)
+            continue
+
+        # then simply call the loaded entry point.
+        extensions[entry_point.name] = obj(app)
+
+
+def load_bundles(app):
+    """"Load bundles as defined in Redash extensions.
+
+    The bundle entry point can be defined as a dotted path to a module
+    or a callable, but it won't be called but just used as a means
+    to find the files under its file system path.
+
+    The name of the directory it looks for files in is "bundle".
+
+    So a Python package with an extension bundle could look like this::
+
+        my_extensions/
+        ├── __init__.py
+        └── wide_footer
+            ├── __init__.py
+            └── bundle
+                ├── extension.js
+                └── styles.css
+
+    and would then need to register the bundle with an entry point
+    under the "redash.periodic_tasks" group, e.g. in your setup.py::
+
+        setup(
+            # ...
+            entry_points={
+                "redash.bundles": [
+                    "wide_footer = my_extensions.wide_footer",
+                ]
+                # ...
+            },
+            # ...
+        )
+
+    """
+    for entry_point in entry_points().get('redash.bundles', []):
+        app.logger.info('Loading Redash bundle "%s".', entry_point.name)
+        module = entry_point_module(entry_point)
+        # Try to get a list of bundle files
+        if not resource_isdir(module, BUNDLE_DIRECTORY):
+            app.logger.error('Redash bundle directory "%s" could not be found.', entry_point.name)
             continue
+        with path(module, BUNDLE_DIRECTORY) as bundle_dir:
+            bundles[entry_point.name] = list(bundle_dir.rglob("*"))
+
+
+def load_periodic_tasks(logger):
+    """Load the periodic tasks as defined in Redash extensions.
+
+    The periodic task entry point needs to return a set of parameters
+    that can be passed to Celery's add_periodic_task:
+
+        https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#entries
+
+    E.g.::
+
+        def add_two_and_two():
+            return {
+                'name': 'add 2 and 2 every 10 seconds'
+                'sig': add.s(2, 2),
+                'schedule': 10.0,  # in seconds or a timedelta
+            }
+
+    and then registered with an entry point under the "redash.periodic_tasks"
+    group, e.g. in your setup.py::
+
+        setup(
+            # ...
+            entry_points={
+                "redash.periodic_tasks": [
+                    "add_two_and_two = calculus.addition:add_two_and_two",
+                ]
+                # ...
+            },
+            # ...
+        )
+    """
+    for entry_point in entry_points().get('redash.periodic_tasks', []):
+        logger.info('Loading periodic Redash tasks "%s" from "%s".', entry_point.name, entry_point.value)
+        try:
+            periodic_tasks[entry_point.name] = entry_point.load()
+        except (ImportError, AttributeError):
+            # and move on if it couldn't load it
+            logger.error('Periodic Redash task "%s" could not be found at "%s".', entry_point.name, entry_point.value)
+
 
-        # Otherwise check if the loaded entry point is a module
-        if isinstance(obj, ModuleType):
-            app.logger.info('Extension %s is a module.', entry_point.name)
-            extensions[entry_point.name]['extension'] = obj
-        # or simply call the loaded entry point instead.
-        else:
-            app.logger.info('Extension %s is a callable.', entry_point.name)
-            extensions[entry_point.name]['extension'] = obj(app)
+def init_app(app):
+    load_extensions(app)
+    load_bundles(app)

redash/worker.py

@@ -8,20 +8,13 @@
 from celery.schedules import crontab
 from celery.signals import worker_process_init
 from celery.utils.log import get_logger
-from importlib_metadata import entry_points
 
-from redash import create_app, settings
+from redash import create_app, extensions, settings
 from redash.metrics import celery as celery_metrics  # noqa
 
 
 logger = get_logger(__name__)
 
-# The periodic Celery tasks as provided by Redash extensions.
-# This is separate from the internal periodic Celery tasks in
-# celery_schedule since the extension task discovery phase is
-# after the configuration has already happened.
-extensions_schedule = {}
-
 
 celery = Celery('redash',
                 broker=settings.CELERY_BROKER,
@@ -92,44 +85,9 @@ def init_celery_flask_app(**kwargs):
 
 @celery.on_after_configure.connect
 def add_periodic_tasks(sender, **kwargs):
-    """Load the periodic tasks as defined in Redash extensions.
-
-    The periodic task entry point needs to return a set of parameters
-    that can be passed to Celery's add_periodic_task:
-
-        https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#entries
-
-    E.g.::
-
-        def add_two_and_two():
-            return {
-                'name': 'add 2 and 2 every 10 seconds'
-                'sig': add.s(2, 2),
-                'schedule': 10.0,  # in seconds or a timedelta
-            }
-
-    and then registered with an entry point under the "redash.periodic_tasks"
-    group, e.g. in your setup.py::
-
-        setup(
-            # ...
-            entry_points={
-                "redash.periodic_tasks": [
-                    "add_two_and_two = calculus.addition:add_two_and_two",
-                ]
-                # ...
-            },
-            # ...
-        )
-    """
-    for entry_point in entry_points().get('redash.periodic_tasks', []):
-        logger.info('Loading periodic Redash tasks %s from %s.', entry_point.name, entry_point.value)
-        try:
-            params = entry_point.load()
-            # Keep a record of our periodic tasks
-            extensions_schedule[entry_point.name] = params
-            # and add it to Celery's periodic task registry, too.
-            sender.add_periodic_task(**params)
-        except (ImportError, AttributeError):
-            # and move on if it couldn't load it
-            logger.error('Periodic Redash task %s could not be found at %s.', entry_point.name, entry_point.value)
+    """Load all periodic tasks from extensions and add them to Celery."""
+    # Populate the redash.extensions.periodic_tasks dictionary
+    extensions.load_periodic_tasks(logger)
+    for params in extensions.periodic_tasks.values():
+        # Add it to Celery's periodic task registry, too.
+        sender.add_periodic_task(**params)