Improve cli

.gitignore

@@ -4,13 +4,11 @@ MANIFEST
 build
 dist
 lib
-quetz_frontend/static
-quetz_frontend/schemas
-quetz_frontend/themes
-quetz_frontend/style.js
-dev_mode/static
-dev_mode/schemas
-dev_mode/themes
+quetz_frontend/app/build
+quetz_frontend/app/static
+quetz_frontend/app/schemas
+quetz_frontend/app/themes
+quetz_frontend/app/style.js
 
 # local env files
 .env.local

.prettierignore

@@ -3,4 +3,4 @@ node_modules
 **/lib
 **/build
 **/static
-quetz_frontend/themes
+quetz_frontend/app/themes

README.md

@@ -23,29 +23,86 @@
 
 ### Development
 
+First of all, clone quetz and quetz-frontend, create a conda environment using the `environment.yml` in quetz, run quetz and modify its config file.
+
 ```bash
-# after installing quetz activate quetz environment
+# Create an environment
+mamba env create -f quetz/environment.yml
 conda activate quetz
+mamba install -c conda-forge nodejs=14 yarn
+```
+
+#### Install Quetz in dev mode
+
+```bash
+cd quetz
+pip install -e .
+
+# Run quetz
+quetz run test_quetz --copy-conf ./dev_config.toml --dev --reload
+```
+
+Modify the `quetz/test_quetz/config.toml` file to add the client_id, client_secret, github username and the front-end paths.
 
-# install the dependencies
-yarn
+```bash
+[github]
+# Register the app here: https://github.com/settings/applications/new
+client_id = "id"
+client_secret = "secret"
 
+[users]
+admins = ["github:username"]
+
+[general]
+frontend_dir = "/path/to/quetz/frontend/quetz_frontend/app/build"
+extensions_dir = "path/to/extensions/folder"
+```
+
+#### Install Quetz-Frontend in dev mode
+
+```bash
 # build the app
 pip install -e .
+
+# Create a link to the quetz folder
+quetz-frontend link-frontend --development
 ```
 
-There is also a watch command to automatically rebuild the application when there are new changes:
+#### Useful commands
 
 ```bash
+# Start an already configured quetz deployment in dev mode:
+quetz start test_quetz --reload
+
+# Build the Quetz-frontend
+yarn run build
+
+# Build the Quetz-Frontend in watch mode
 yarn run watch
 ```
 
-Remember to add the path to the frontend build folder in the quetz `config.toml` file and add your github user as an admin.
+### Command line tool
+
+Quetz fronted also comes with a cli to manage extensions
 
 ```bash
-[users]
-admins = ["github:username"]
+Usage: quetz-frontend [OPTIONS] COMMAND [ARGS]...
+
+Options:
+  --install-completion  Install completion for the current shell.
+  --show-completion     Show completion for the current shell, to copy it or
+                        customize the installation.
+  --help                Show this message and exit.
+
+Commands:
+  build           Build an extension
+  clean           Clean the extensions directory
+  clean-frontend  Clean the Quetz-Frontend
+  develop         Build and install an extension in dev mode
+  install         Build and install an extension
+  link-frontend   Intall the Quetz-Frontend
+  list            List of extensions
+  paths
+  watch           Watch an extension
 
-[general]
-frontend_dir = "/path/to/quetz-frontend/quetz_frontend/app/build"
 ```

package.json

@@ -29,17 +29,17 @@
   "devDependencies": {
     "@typescript-eslint/eslint-plugin": "^4.28.5",
     "@typescript-eslint/parser": "^4.28.5",
-    "eslint": "^7.22.0",
+    "eslint": "^7.32.0",
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-jest": "^24.4.0",
-    "eslint-plugin-jsdoc": "^36.0.6",
-    "eslint-plugin-prettier": "^3.4.0",
-    "eslint-plugin-react": "^7.24.0",
+    "eslint-plugin-jsdoc": "^36.1.0",
+    "eslint-plugin-prettier": "^4.0.0",
+    "eslint-plugin-react": "^7.25.1",
     "lerna": "^4.0.0",
-    "lint-staged": "^11.0.1",
+    "lint-staged": "^11.1.2",
     "npm-run-all": "^4.1.5",
-    "prettier": "^2.3.2",
+    "prettier": "^2.4.0",
     "shell-quote": "^1.7.2",
-    "yarn": "1.22.10"
+    "yarn": "1.22.11"
   }
 }

pyproject.toml

@@ -3,15 +3,15 @@ requires = ["jupyter_packaging~=0.10,<2", "jupyterlab~=3.0"]
 build-backend = "jupyter_packaging.build_api"
 
 [tool.jupyter-packaging.options]
-skip-if-exists = ["quetz_frontend/app/build/style.js"]
-ensured-targets = ["quetz_frontend/app/build/style.js"] #, "jupyterlab_zethus/labextension/package.json"]
+skip-if-exists = ["quetz_frontend/app/style.js"]
+ensured-targets = ["quetz_frontend/app/style.js"]
 
 [tool.jupyter-packaging.builder]
 factory = "jupyter_packaging.npm_builder"
 
 [tool.jupyter-packaging.build-args]
 build_cmd = "build:prod"
-npm = ["jlpm"]
+npm = ["yarn"]
 
 [tool.check-manifest]
 ignore = ["quetz_frontend/app/build/**", "yarn.lock", ".*", "package-lock.json"]

quetz_frontend/app/index.js

@@ -8,7 +8,7 @@ import { App } from '@quetz-frontend/application';
 
 import { PageConfig } from '@jupyterlab/coreutils';
 
-import './build/style.js';
+import './style.js';
 
 // Promise.allSettled polyfill, until our supported browsers implement it
 // See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/allSettled

quetz_frontend/app/package.json

@@ -10,7 +10,7 @@
     "build:prod:release": "webpack --config webpack.prod.release.config.js",
     "build:prod:stats": "webpack --profile --config webpack.prod.minimize.config.js --json > stats.json",
     "build:stats": "webpack --profile --json > stats.json",
-    "clean": "rimraf build",
+    "clean": "rimraf build && rimraf static && rimraf schemas && rimraf themes && rimraf style.js",
     "clean:all": "yarn run clean && rimraf node_modules",
     "prepublishOnly": "npm run build",
     "watch": "webpack --config ./webpack.config.watch.js"
@@ -96,7 +96,7 @@
   "jupyterlab": {
     "name": "Quetz-Frontend",
     "version": "0.1.0",
-    "outputDir": "./out",
+    "outputDir": ".",
     "buildDir": "./build",
     "staticDir": "./static",
     "extensions": [

quetz_frontend/app/webpack.config.js

@@ -1,14 +1,21 @@
 // Copyright (c) Jupyter Development Team.
 // Distributed under the terms of the Modified BSD License.
+const path = require('path');
 const CopyPlugin = require('copy-webpack-plugin');
 const HtmlWebpackPlugin = require('html-webpack-plugin');
 const webpack = require('webpack');
 const { ModuleFederationPlugin } = webpack.container;
+const WPPlugin = require('@jupyterlab/builder').WPPlugin;
 const Build = require('@jupyterlab/builder').Build;
 
 const packageData = require('./package.json');
 const jlab = packageData.jupyterlab;
 
+// Directories
+const staticDir = path.resolve(jlab.staticDir);
+const outputDir = path.resolve(jlab.outputDir);
+const buildDir = path.resolve(jlab.buildDir);
+
 // Create a list of application extensions and mime extensions from
 // jlab.extensions
 const extensions = {};
@@ -29,7 +36,7 @@ for (const key of jlab.extensions) {
 // such as setting schema files, theme assets, etc.
 const extensionAssetConfig = Build.ensureAssets({
   packageNames: jlab.extensions,
-  output: './build',
+  output: outputDir,
 });
 
 /**
@@ -139,7 +146,7 @@ module.exports = [
       './bootstrap.js',
     ],
     output: {
-      path: __dirname + '/build',
+      path: buildDir,
       filename: 'bundle.js',
     },
     bail: false,
@@ -205,6 +212,8 @@ module.exports = [
         inject: false,
         filename: 'index.html.j2',
       }),
+      // custom plugin that copies the assets to the static directory
+      new WPPlugin.FrontEndPlugin(buildDir, staticDir),
       new ModuleFederationPlugin({
         library: {
           type: 'var',

quetz_frontend/backend.py

@@ -1,5 +1,4 @@
 import os
-import sys
 import json
 import glob
 import copy
@@ -18,6 +17,7 @@
 from quetz.authentication.registry import AuthenticatorRegistry
 from quetz import authorization, rest_models
 
+from .paths import LOCAL_APP_DIR, GLOBAL_FRONTEND_DIR, GLOBAL_EXTENSIONS_DIR
 
 logger = logging.getLogger('quetz.frontend')
 config = Config()
@@ -30,20 +30,27 @@
 frontend_settings = {}
 federated_extensions = []
 
-GLOBAL_FRONTEND_DIR = pjoin(sys.prefix, '/share/quetz/frontend/')
-GLOBAL_EXTENSIONS_DIR = pjoin(sys.prefix, '/share/quetz/frontend/extensions/')
-HERE = os.path.abspath(os.path.dirname(__file__))
-LOCAL_FRONTEND_DIR = os.path.join(HERE, "app", "build")
+frontend_dir = None
+extensions_dir = None
 
-if os.path.exists(LOCAL_FRONTEND_DIR):
-    frontend_dir = LOCAL_FRONTEND_DIR
+if os.path.exists(LOCAL_APP_DIR):
+    frontend_dir = LOCAL_APP_DIR
     logger.info("Using local DEVELOPMENT frontend directory.")
 elif os.path.exists(GLOBAL_FRONTEND_DIR):
     frontend_dir = GLOBAL_FRONTEND_DIR
+    logger.info("Using global frontend directory.")
 else:
-    raise RuntimeException(f"Could not find frontend files in:\n- {LOCAL_FRONTEND_DIR}\n- {GLOBAL_FRONTEND_DIR}")
+    raise RuntimeException(f"Could not find frontend files in:\n- {LOCAL_APP_DIR}\n- {GLOBAL_FRONTEND_DIR}")
+
+if os.path.exists(GLOBAL_EXTENSIONS_DIR):
+    extensions_dir = GLOBAL_EXTENSIONS_DIR
+    logger.info("Using global frontend extensions directory.")
+else :
+    os.mkdir(GLOBAL_EXTENSIONS_DIR)
+    logger.info("Creating a global frontend extensions directory.")
 
 logger.info(f"Successfully found frontend in {frontend_dir}")
+logger.info(f"Successfully found frontend extensions in {extensions_dir}")
 
 @mock_router.get('/api/sessions', include_in_schema=False)
 def mock_sessions():
@@ -77,6 +84,9 @@ def extensions(
     auth: authorization.Rules = Depends(get_rules),
 ):
     path = pjoin(extensions_dir, resource)
+    logger.warn(path)
+    logger.warn(os.path.exists(path))
+    logger.warn(under_frontend_dir(path))
     if os.path.exists(path) and under_frontend_dir(path):
         return FileResponse(path=path)
     else:
@@ -89,7 +99,7 @@ def static(
     dao: Dao = Depends(get_dao),
     auth: authorization.Rules = Depends(get_rules),
 ):
-    path = pjoin(frontend_dir, resource)
+    path = pjoin(frontend_dir, 'static', resource)
     if os.path.exists(path) and under_frontend_dir(path):
         return FileResponse(path=path)
     else:
@@ -102,6 +112,7 @@ def index(
     dao: Dao = Depends(get_dao),
     auth: authorization.Rules = Depends(get_rules),
 ):
+    global config_data
     user_id = auth.get_user()
     profile = dao.get_profile(user_id)
 
@@ -112,20 +123,24 @@ def index(
             else resource.split('/')[-1]
         )
 
-        path = pjoin(frontend_dir, file_name)
+        path = pjoin(frontend_dir, 'static', file_name)
         if os.path.exists(path) and under_frontend_dir(path) :
             return FileResponse(path=path)
         else:
             raise HTTPException(status_code=404)
     else:
+        extensions = get_federated_extensions([extensions_dir])
+        federated_extensions = load_federated_extensions(extensions)
+        config_data['federated_extensions'] = federated_extensions
+
         if profile :
             index_rendered = get_rendered_index(config_data, profile, index_template)
             return HTMLResponse(content=index_rendered, status_code=200)
         else:
-            index_html_path = pjoin(frontend_dir, "index.html")
+            index_html_path = pjoin(frontend_dir, 'static', "index.html")
             if not os.path.exists(index_html_path):
                 render_index()
-            return FileResponse(path=pjoin(frontend_dir, "index.html"))
+            return FileResponse(path=pjoin(frontend_dir, 'static', "index.html"))
 
 def under_frontend_dir(path):
     """
@@ -136,7 +151,8 @@ def under_frontend_dir(path):
     """
     path = os.path.abspath(path)
     fdir = os.path.abspath(frontend_dir)
-    return os.path.commonpath([path, fdir]) == fdir
+    globalDir = os.path.abspath(GLOBAL_FRONTEND_DIR)
+    return os.path.commonpath([path, fdir]) == fdir or os.path.commonpath([path, globalDir]) == globalDir
 
 
 def get_rendered_index(config_data, profile, index_template):
@@ -154,12 +170,12 @@ def render_index():
         del cfg["logged_in_user_profile"]
 
     # Create index.html with config
-    with open(pjoin(frontend_dir, "index.html.j2")) as fi:
+    with open(pjoin(frontend_dir, 'static', "index.html.j2")) as fi:
         index_template = jinja2.Template(fi.read())
-    with open(pjoin(frontend_dir, "index.html"), "w") as fo:
+    with open(pjoin(frontend_dir, 'static', "index.html"), "w") as fo:
         fo.write(index_template.render(page_config=config_data))
 
-    template_path = pjoin(frontend_dir, "..", "templates")
+    template_path = pjoin(frontend_dir, "templates")
     if os.path.exists(template_path):
         # Load settings
         with open(pjoin(template_path, "settings.json")) as fi:
@@ -217,7 +233,7 @@ def register(app):
     app.include_router(catchall_router)
 
     frontend_dir = config.general_frontend_dir
-    extensions_dir = GLOBAL_EXTENSIONS_DIR
+    #extensions_dir = config.general_extensions_dir
 
     logger.info(f"Configured frontend found: {frontend_dir}")
     logger.info(f"Configured extensions directory: {extensions_dir}")