Decouple extensions from Flask app.

This separates the extension registry from the Flask app and also introduces a separate registry for preriodic tasks.

Fix #3466.

bin/bundle-extensions

@@ -1,11 +1,10 @@
 #!/usr/bin/env python
 
 import os
-from subprocess import call
 from distutils.dir_util import copy_tree
 
-from pkg_resources import iter_entry_points, resource_filename, resource_isdir
-
+import importlib_metadata
+import importlib_resources
 
 
 # Make a directory for extensions and set it as an environment variable
@@ -19,21 +18,28 @@ if not os.path.exists(EXTENSIONS_DIRECTORY):
     os.makedirs(EXTENSIONS_DIRECTORY)
 os.environ["EXTENSIONS_DIRECTORY"] = EXTENSIONS_RELATIVE_PATH
 
-for entry_point in iter_entry_points('redash.extensions'):
-    # This is where the frontend code for an extension lives
-    # inside of its package.
-    content_folder_relative = os.path.join(
-        entry_point.name, 'bundle')
-    (root_module, _) = os.path.splitext(entry_point.module_name)
 
-    if not resource_isdir(root_module, content_folder_relative):
-        continue
+def is_resource_dir(module, resource):
+    try:
+        return (
+            resource in importlib_resources.contents(module)
+            and not importlib_resources.is_resource(module, resource)
+        )
+    except TypeError:
+        # module isn't a package, so can't have a subdirectory/-package
+        return False
 
-    content_folder = resource_filename(root_module, content_folder_relative)
 
-    # This is where we place our extensions folder.
-    destination = os.path.join(
-        EXTENSIONS_DIRECTORY,
-        entry_point.name)
+for entry_point in importlib_metadata.entry_points().get('redash.extensions', []):
+
+    # Check if there is a "bundle" subdirectory/-package in the
+    # entrypoint's module and ignoring the entrypoint if not.
+    module_name = entry_point.pattern.match(entry_point.value).group('module')
+    if not is_resource_dir(module_name, 'bundle'):
+        continue
 
-    copy_tree(content_folder, destination)
+    with importlib_resources.path(module_name, 'bundle') as bundle_dir:
+        # Copy content of extension bundle into extensions directory
+        destination = os.path.join(EXTENSIONS_DIRECTORY, entry_point.name)
+        print("Copying {} to {}".format(bundle_dir, destination))
+        copy_tree(str(bundle_dir), destination)

redash/extensions.py

@@ -1,30 +1,50 @@
-import os
-from pkg_resources import iter_entry_points, resource_isdir, resource_listdir
+from importlib_metadata import entry_points
+
+# The global Redash extension registry
+extensions = {}
+
+# The periodic Celery task registry
+periodic_tasks = {}
 
 
 def init_extensions(app):
-    """
-    Load the Redash extensions for the given Redash Flask app.
-    """
-    if not hasattr(app, 'redash_extensions'):
-        app.redash_extensions = {}
+    """Load the Redash extensions for the given Redash Flask app.
 
-    for entry_point in iter_entry_points('redash.extensions'):
+    The extension entry pooint can return any type of value but
+    must take a Flask application object.
+
+    E.g.::
+
+        def extension(app):
+            app.logger.info("Loading the Foobar extenions")
+            Foobar(app)
+
+    """
+    for entry_point in entry_points().get('redash.extensions', []):
         app.logger.info('Loading Redash extension %s.', entry_point.name)
-        try:
-            extension = entry_point.load()
-            app.redash_extensions[entry_point.name] = {
-                "entry_function": extension(app),
-                "resources_list": []
+        init_extension = entry_point.load()
+        extensions[entry_point.name] = init_extension(app)
+
+
+def init_periodic_tasks(app):
+    """Load the Redash extensions for the given Redash Flask app.
+
+    The periodic task entry point needs to return a set of parameters
+    that can be passed to Celery's add_periodic_task:
+
+        https://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html#entries
+
+    E.g.::
+
+        def periodic_task():
+            return {
+                'name': 'add 2 and 2 every 10 seconds'
+                'sig': add.s(2, 2),
+                'schedule': 10.0,
             }
-        except ImportError:
-            app.logger.info('%s does not have a callable and will not be loaded.', entry_point.name)
-            (root_module, _) = os.path.splitext(entry_point.module_name)
-            content_folder_relative = os.path.join(entry_point.name, 'bundle')
-
-            # If it's a frontend extension only, store a list of files in the bundle directory.
-            if resource_isdir(root_module, content_folder_relative):
-                app.redash_extensions[entry_point.name] = {
-                    "entry_function": None,
-                    "resources_list": resource_listdir(root_module, content_folder_relative)
-                }
+
+    """
+    for entry_point in entry_points().get('redash.periodic_tasks', []):
+        app.logger.info('Loading Redash periodic tasks %s.', entry_point.name)
+        init_periodic_task = entry_point.load()
+        periodic_tasks[entry_point.name] = init_periodic_task()

redash/worker.py

@@ -8,7 +8,7 @@
 from celery import Celery
 from celery.schedules import crontab
 from celery.signals import worker_process_init
-from redash import create_app, settings
+from redash import create_app, extensions, settings
 from redash.metrics import celery as celery_metrics
 
 celery = Celery('redash',
@@ -74,11 +74,8 @@ def init_celery_flask_app(**kwargs):
     app.app_context().push()
 
 
-# Commented until https://github.com/getredash/redash/issues/3466 is implemented.
 # Hook for extensions to add periodic tasks.
-# @celery.on_after_configure.connect
-# def add_periodic_tasks(sender, **kwargs):
-#     app = safe_create_app()
-#     periodic_tasks = getattr(app, 'periodic_tasks', {})
-#     for params in periodic_tasks.values():
-#         sender.add_periodic_task(**params)
+@celery.on_after_configure.connect
+def add_periodic_tasks(sender, **kwargs):
+    for params in extensions.periodic_tasks.values():
+        sender.add_periodic_task(**params)

requirements.txt

@@ -61,3 +61,5 @@ disposable-email-domains
 # It is not included by default because of the GPL license conflict.
 # ldap3==2.2.4
 gevent==1.4.0
+importlib-metadata==0.8
+importlib_resources==1.0.2