diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9e6f1e48c..beb20c8be 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -28,7 +28,7 @@ updates: prefix: npm dependencies (development) include: scope - package-ecosystem: npm - directory: betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/ + directory: betty/extension/cotton_candy/webpack/ schedule: interval: weekly assignees: @@ -37,7 +37,7 @@ updates: prefix: npm dependencies (Cotton Candy extension) include: scope - package-ecosystem: npm - directory: betty/extension/http_api_doc/assets/betty.extension.npm._Npm/src/ + directory: betty/extension/http_api_doc/webpack/ schedule: interval: weekly assignees: @@ -46,7 +46,7 @@ updates: prefix: npm dependencies (HttpApiDoc extension) include: scope - package-ecosystem: npm - directory: betty/extension/maps/assets/betty.extension.npm._Npm/src/ + directory: betty/extension/maps/webpack/ schedule: interval: weekly assignees: @@ -55,7 +55,7 @@ updates: prefix: npm dependencies (Maps extension) include: scope - package-ecosystem: npm - directory: betty/extension/trees/assets/betty.extension.npm._Npm/src/ + directory: betty/extension/trees/webpack/ schedule: interval: weekly assignees: diff --git a/.gitignore b/.gitignore index 5a5d8e430..6974e7772 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ __pycache__ /betty.egg-info /build /dist +/prebuild *.pyc *.pyc.* diff --git a/.stylelintignore b/.stylelintignore index 6bde87fef..f01501f4d 100644 --- a/.stylelintignore +++ b/.stylelintignore @@ -1 +1 @@ -betty/**/assets/betty.extension.npm._Npm/build/* +prebuild/**/* diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 000000000..670c1e756 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +prune node_modules diff --git a/betty/_npm.py b/betty/_npm.py new file mode 100644 index 000000000..06f5854a4 --- /dev/null +++ b/betty/_npm.py @@ -0,0 +1,81 @@ +""" +Provide tools to integrate extensions with `npm `_. + +This module is internal. +""" + +from __future__ import annotations + +import logging +import sys +from asyncio import subprocess as aiosubprocess +from pathlib import Path +from typing import Sequence + +from betty.asyncio import wait_to_thread +from betty.error import UserFacingError +from betty.locale import Str, DEFAULT_LOCALIZER +from betty.requirement import Requirement +from betty.subprocess import run_process + + +_NPM_UNAVAILABLE_MESSAGE = Str._( + "npm (https://www.npmjs.com/) must be available for features that require Node.js packages to be installed. Ensure that the `npm` executable is available in your `PATH`." +) + + +class NpmUnavailable(UserFacingError, RuntimeError): + def __init__(self): + super().__init__(_NPM_UNAVAILABLE_MESSAGE) + + +async def npm( + arguments: Sequence[str], + cwd: Path | None = None, +) -> aiosubprocess.Process: + """ + Run an npm command. + """ + try: + return await run_process( + ["npm", *arguments], + cwd=cwd, + # Use a shell on Windows so subprocess can find the executables it needs (see + # https://bugs.python.org/issue17023). + shell=sys.platform.startswith("win32"), + ) + except FileNotFoundError: + raise NpmUnavailable() + + +class NpmRequirement(Requirement): + def __init__(self): + super().__init__() + self._met: bool + self._summary: Str + self._details = _NPM_UNAVAILABLE_MESSAGE + + def _check(self) -> None: + if hasattr(self, "_met"): + return + try: + wait_to_thread(npm(["--version"])) + except NpmUnavailable: + self._met = False + self._summary = Str._("`npm` is not available") + else: + self._met = True + self._summary = Str._("`npm` is available") + finally: + logging.getLogger(__name__).debug(self._summary.localize(DEFAULT_LOCALIZER)) + + def is_met(self) -> bool: + self._check() + return self._met + + def summary(self) -> Str: + self._check() + return self._summary + + def details(self) -> Str: + return self._details diff --git a/betty/_package/pyinstaller/__init__.py b/betty/_package/pyinstaller/__init__.py index 8f5d59e3f..6cdc6b2d7 100644 --- a/betty/_package/pyinstaller/__init__.py +++ b/betty/_package/pyinstaller/__init__.py @@ -10,33 +10,31 @@ from betty._package.pyinstaller.hooks import HOOKS_DIRECTORY_PATH from betty.app import App -from betty.app.extension import discover_extension_types, Extension -from betty.asyncio import gather -from betty.extension.npm import _Npm, build_assets, _NpmBuilder +from betty.app.extension import discover_extension_types +from betty.extension.webpack import ( + Webpack, + WebpackEntrypointProvider, +) from betty.fs import ROOT_DIRECTORY_PATH -from betty.project import ExtensionConfiguration +from betty.job import Context -async def _build_assets() -> None: - npm_builder_extension_types: list[type[_NpmBuilder & Extension]] = [ - extension_type - for extension_type in discover_extension_types() - if issubclass(extension_type, _NpmBuilder) - ] +async def prebuild_webpack_assets() -> None: + """ + Prebuild Webpack assets for inclusion in package builds. + """ + job_context = Context() async with App.new_temporary() as app, app: - app.project.configuration.extensions.append(ExtensionConfiguration(_Npm)) - for extension_type in npm_builder_extension_types: - app.project.configuration.extensions.append( - ExtensionConfiguration(extension_type) - ) - await gather( - *( - [ - build_assets(app.extensions[extension_type]) # type: ignore[arg-type] - for extension_type in npm_builder_extension_types - ] - ) + app.project.configuration.extensions.enable(Webpack) + webpack = app.extensions[Webpack] + app.project.configuration.extensions.enable( + *{ + extension_type + for extension_type in discover_extension_types() + if issubclass(extension_type, WebpackEntrypointProvider) + } ) + await webpack.prebuild(job_context=job_context) async def a_pyz_exe_coll() -> tuple[Analysis, PYZ, EXE, COLLECT]: @@ -52,12 +50,18 @@ async def a_pyz_exe_coll() -> tuple[Analysis, PYZ, EXE, COLLECT]: else: raise RuntimeError(f"Unsupported platform {sys.platform}.") - await _build_assets() + await prebuild_webpack_assets() block_cipher = None datas = [] data_file_path_patterns = [ + # Assets. "betty/assets/**", "betty/extension/*/assets/**", + # Webpack. + ".browserslistrc", + "betty/extension/*/webpack/**", + "tsconfig.json", + "prebuild/**", ] for data_file_path_pattern in data_file_path_patterns: for data_file_path_str in glob( diff --git a/betty/assets/betty.pot b/betty/assets/betty.pot index dd3767909..b8a52132f 100644 --- a/betty/assets/betty.pot +++ b/betty/assets/betty.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Betty VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2024-03-27 23:37+0000\n" +"POT-Creation-Date: 2024-04-30 22:56+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -189,16 +189,7 @@ msgstr "" msgid "Birth" msgstr "" -msgid "Built the Cotton Candy front-end assets." -msgstr "" - -msgid "Built the HTTP API documentation." -msgstr "" - -msgid "Built the interactive family trees." -msgstr "" - -msgid "Built the interactive maps." +msgid "Built the Webpack front-end assets." msgstr "" msgid "Burial" @@ -590,10 +581,10 @@ msgstr "" msgid "Places" msgstr "" -msgid "Pre-built assets" +msgid "Pre-built Webpack front-end assets are available" msgstr "" -msgid "Pre-built assets are unavailable for {extension_names}." +msgid "Pre-built Webpack front-end assets are unavailable" msgstr "" msgid "Presence" diff --git a/betty/assets/locale/de-DE/betty.po b/betty/assets/locale/de-DE/betty.po index ac42bbac5..5a34f2701 100644 --- a/betty/assets/locale/de-DE/betty.po +++ b/betty/assets/locale/de-DE/betty.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: Betty VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2024-03-27 23:37+0000\n" +"POT-Creation-Date: 2024-04-30 22:56+0100\n" "PO-Revision-Date: 2024-02-08 13:24+0000\n" "Last-Translator: Bart Feenstra \n" "Language: de\n" @@ -279,17 +279,8 @@ msgstr "Betty-Projektkonfiguration ({supported_formats})" msgid "Birth" msgstr "Geburt" -msgid "Built the Cotton Candy front-end assets." -msgstr "Cotton Candy front-end assets erstellt." - -msgid "Built the HTTP API documentation." -msgstr "Die HTTP-API-Dokumentation wurde erstellt." - -msgid "Built the interactive family trees." -msgstr "Die interaktiven Stammbäume sind erstellt." - -msgid "Built the interactive maps." -msgstr "Die interaktiven Karten wurden erstellt." +msgid "Built the Webpack front-end assets." +msgstr "Webpack front-end assets erstellt." msgid "Burial" msgstr "Beerdigung" @@ -760,11 +751,11 @@ msgstr "Ort" msgid "Places" msgstr "Orte" -msgid "Pre-built assets" -msgstr "Vorgefertigte Objekte" +msgid "Pre-built Webpack front-end assets are available" +msgstr "Vorgefertigte Webpack front-end assets sind verfügbar" -msgid "Pre-built assets are unavailable for {extension_names}." -msgstr "Vorgefertigte Objekte für {extension_names} sind nicht verfügbar." +msgid "Pre-built Webpack front-end assets are unavailable" +msgstr "Vorgefertigte Webpack front-end assets sind nicht verfügbar" msgid "Presence" msgstr "Anwesenheit" diff --git a/betty/assets/locale/fr-FR/betty.po b/betty/assets/locale/fr-FR/betty.po index d42baadbe..c12b69373 100644 --- a/betty/assets/locale/fr-FR/betty.po +++ b/betty/assets/locale/fr-FR/betty.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2024-03-27 23:37+0000\n" +"POT-Creation-Date: 2024-04-30 22:56+0100\n" "PO-Revision-Date: 2024-02-08 13:24+0000\n" "Last-Translator: Bart Feenstra \n" "Language: fr\n" @@ -233,16 +233,7 @@ msgstr "" msgid "Birth" msgstr "Naissance" -msgid "Built the Cotton Candy front-end assets." -msgstr "" - -msgid "Built the HTTP API documentation." -msgstr "" - -msgid "Built the interactive family trees." -msgstr "" - -msgid "Built the interactive maps." +msgid "Built the Webpack front-end assets." msgstr "" msgid "Burial" @@ -674,10 +665,10 @@ msgstr "Lieu" msgid "Places" msgstr "Lieux" -msgid "Pre-built assets" +msgid "Pre-built Webpack front-end assets are available" msgstr "" -msgid "Pre-built assets are unavailable for {extension_names}." +msgid "Pre-built Webpack front-end assets are unavailable" msgstr "" msgid "Presence" diff --git a/betty/assets/locale/nl-NL/betty.po b/betty/assets/locale/nl-NL/betty.po index 7664c9c96..e20068122 100644 --- a/betty/assets/locale/nl-NL/betty.po +++ b/betty/assets/locale/nl-NL/betty.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2024-03-27 23:37+0000\n" +"POT-Creation-Date: 2024-04-30 22:56+0100\n" "PO-Revision-Date: 2024-02-11 15:31+0000\n" "Last-Translator: Bart Feenstra \n" "Language: nl\n" @@ -276,17 +276,8 @@ msgstr "Betty-projectconfiguratie ({supported_formats})" msgid "Birth" msgstr "Geboorte" -msgid "Built the Cotton Candy front-end assets." -msgstr "De Cotton Candy front-endassets gegenereerd." - -msgid "Built the HTTP API documentation." -msgstr "HTTP API-documentatie gegenereerd." - -msgid "Built the interactive family trees." -msgstr "De interactieve stambomen gegenereerd." - -msgid "Built the interactive maps." -msgstr "De interactieve kaarten gegenereerd." +msgid "Built the Webpack front-end assets." +msgstr "De Webpack front-end assets gebouwd." msgid "Burial" msgstr "Begravenis" @@ -755,11 +746,11 @@ msgstr "Plaats" msgid "Places" msgstr "Plaatsen" -msgid "Pre-built assets" -msgstr "Kant-en-klare assets" +msgid "Pre-built Webpack front-end assets are available" +msgstr "Vooraf gebouwde Webpack front-end assets zijn beschikbaar." -msgid "Pre-built assets are unavailable for {extension_names}." -msgstr "Kant-en-klare assets zijn beschikbaar voor {extension_names}." +msgid "Pre-built Webpack front-end assets are unavailable" +msgstr "Vooraf gebouwde Webpack front-end assets zijn niet beschikbaar." msgid "Presence" msgstr "Aanwezigheid" diff --git a/betty/assets/locale/uk/betty.po b/betty/assets/locale/uk/betty.po index b926d05df..6cce4c192 100644 --- a/betty/assets/locale/uk/betty.po +++ b/betty/assets/locale/uk/betty.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: Betty VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2024-03-27 23:37+0000\n" +"POT-Creation-Date: 2024-04-30 22:56+0100\n" "PO-Revision-Date: 2024-02-08 13:08+0000\n" "Last-Translator: Rainer Thieringer \n" "Language: uk\n" @@ -233,16 +233,7 @@ msgstr "" msgid "Birth" msgstr "Народження" -msgid "Built the Cotton Candy front-end assets." -msgstr "" - -msgid "Built the HTTP API documentation." -msgstr "" - -msgid "Built the interactive family trees." -msgstr "" - -msgid "Built the interactive maps." +msgid "Built the Webpack front-end assets." msgstr "" msgid "Burial" @@ -674,10 +665,10 @@ msgstr "Місце" msgid "Places" msgstr "Місця" -msgid "Pre-built assets" +msgid "Pre-built Webpack front-end assets are available" msgstr "" -msgid "Pre-built assets are unavailable for {extension_names}." +msgid "Pre-built Webpack front-end assets are unavailable" msgstr "" msgid "Presence" diff --git a/betty/assets/templates/head.html.j2 b/betty/assets/templates/head.html.j2 index da0dadd47..f95989174 100644 --- a/betty/assets/templates/head.html.j2 +++ b/betty/assets/templates/head.html.j2 @@ -41,9 +41,3 @@ {% endif %} {% endif %} -{% for css_path in public_css_paths %} - -{% endfor %} -{% for js_path in public_js_paths %} - -{% endfor %} \ No newline at end of file diff --git a/betty/assets/templates/scripts.html.j2 b/betty/assets/templates/scripts.html.j2 new file mode 100644 index 000000000..ccb385c9d --- /dev/null +++ b/betty/assets/templates/scripts.html.j2 @@ -0,0 +1,3 @@ +{% for js_path in public_js_paths | unique %} + +{% endfor %} diff --git a/betty/assets/templates/stylesheets.html.j2 b/betty/assets/templates/stylesheets.html.j2 new file mode 100644 index 000000000..2bc504aad --- /dev/null +++ b/betty/assets/templates/stylesheets.html.j2 @@ -0,0 +1,3 @@ +{% for css_path in public_css_paths | unique %} + +{% endfor %} diff --git a/betty/extension/__init__.py b/betty/extension/__init__.py index 3da998af8..fad171b1c 100644 --- a/betty/extension/__init__.py +++ b/betty/extension/__init__.py @@ -11,6 +11,7 @@ from betty.extension.nginx import Nginx from betty.extension.privatizer import Privatizer from betty.extension.trees import Trees +from betty.extension.webpack import Webpack from betty.extension.wikipedia import Wikipedia __all__ = ( @@ -23,5 +24,6 @@ "Nginx", "Privatizer", "Trees", + "Webpack", "Wikipedia", ) diff --git a/betty/extension/cotton_candy/__init__.py b/betty/extension/cotton_candy/__init__.py index a583cba8b..536f41b90 100644 --- a/betty/extension/cotton_candy/__init__.py +++ b/betty/extension/cotton_candy/__init__.py @@ -4,27 +4,23 @@ from __future__ import annotations -import asyncio -import logging import re from collections import defaultdict from collections.abc import Sequence, AsyncIterable from pathlib import Path -from shutil import copy2 from typing import Any, Callable, Iterable, Self, cast from PyQt6.QtWidgets import QWidget -from aiofiles.os import makedirs from jinja2 import pass_context from jinja2.runtime import Context from betty.app.extension import ConfigurableExtension, Extension, Theme from betty.config import Configuration from betty.extension.cotton_candy.search import Index -from betty.extension.npm import _Npm, _NpmBuilder, npm +from betty.extension.webpack import Webpack, WebpackEntrypointProvider from betty.functools import walk -from betty.generate import Generator, GenerationContext from betty.gui import GuiBuilder +from betty.html import CssProvider from betty.jinja2 import ( Jinja2Provider, context_app, @@ -221,11 +217,11 @@ def dump(self) -> VoidableDump: class CottonCandy( Theme, + CssProvider, ConfigurableExtension[CottonCandyConfiguration], - Generator, GuiBuilder, - _NpmBuilder, Jinja2Provider, + WebpackEntrypointProvider, ): @classmethod def name(cls) -> str: @@ -233,12 +229,39 @@ def name(cls) -> str: @classmethod def depends_on(cls) -> set[type[Extension]]: - return {_Npm} + return {Webpack} @classmethod - def assets_directory_path(cls) -> Path | None: + def comes_after(cls) -> set[type[Extension]]: + from betty.extension import Maps, Trees + + return {Maps, Trees} + + @classmethod + def assets_directory_path(cls) -> Path: return Path(__file__).parent / "assets" + @classmethod + def webpack_entrypoint_directory_path(cls) -> Path: + return Path(__file__).parent / "webpack" + + def webpack_entrypoint_cache_keys(self) -> Sequence[str]: + return ( + self._app.project.configuration.root_path, + self._configuration.primary_inactive_color.hex, + self._configuration.primary_active_color.hex, + self._configuration.link_inactive_color.hex, + self._configuration.link_active_color.hex, + ) + + @property + def public_css_paths(self) -> list[str]: + return [ + self.app.static_url_generator.generate( + "css/betty.extension.CottonCandy.css" + ), + ] + @classmethod def label(cls) -> Str: return Str.plain("Cotton Candy") @@ -271,40 +294,6 @@ def filters(self) -> dict[str, Callable[..., Any]]: "person_descendant_families": person_descendant_families, } - async def npm_build( - self, working_directory_path: Path, assets_directory_path: Path - ) -> None: - await self.app.extensions[_Npm].install(type(self), working_directory_path) - await npm(("run", "webpack"), cwd=working_directory_path) - await self._copy_npm_build( - working_directory_path / "webpack-build", assets_directory_path - ) - logging.getLogger(__name__).info( - self._app.localizer._("Built the Cotton Candy front-end assets.") - ) - - async def _copy_npm_build( - self, source_directory_path: Path, destination_directory_path: Path - ) -> None: - await makedirs(destination_directory_path, exist_ok=True) - await asyncio.to_thread( - copy2, - source_directory_path / "cotton_candy.css", - destination_directory_path / "cotton_candy.css", - ) - await asyncio.to_thread( - copy2, - source_directory_path / "cotton_candy.js", - destination_directory_path / "cotton_candy.js", - ) - - async def generate(self, job_context: GenerationContext) -> None: - assets_directory_path = await self.app.extensions[_Npm].ensure_assets(self) - await makedirs(self.app.project.configuration.www_directory_path, exist_ok=True) - await self._copy_npm_build( - assets_directory_path, self.app.project.configuration.www_directory_path - ) - @pass_context async def _global_search_index(context: Context) -> AsyncIterable[dict[str, str]]: diff --git a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/.gitignore b/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/.gitignore deleted file mode 100644 index 796b96d1c..000000000 --- a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build diff --git a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/main.ts b/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/main.ts deleted file mode 100644 index 9b17660e1..000000000 --- a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/main.ts +++ /dev/null @@ -1,6 +0,0 @@ -'use strict' - -import './file.ts' // eslint-disable-line no-unused-vars -import './search.ts' // eslint-disable-line no-unused-vars -import './show.ts' // eslint-disable-line no-unused-vars -import './main.scss' // eslint-disable-line @typescript-eslint/no-unused-vars diff --git a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/webpack.config.js b/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/webpack.config.js deleted file mode 100644 index a598af181..000000000 --- a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/webpack.config.js +++ /dev/null @@ -1,98 +0,0 @@ -'use strict' - -import { CleanWebpackPlugin } from 'clean-webpack-plugin' -import MiniCssExtractPlugin from 'mini-css-extract-plugin' -import path from 'path' -import { readFile } from 'node:fs/promises' -import url from 'node:url' - -const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) -const configuration = JSON.parse(await readFile('./webpack.config.json')) - -export default { - mode: configuration.debug ? 'development' : 'production', - entry: { - cotton_candy: path.resolve(__dirname, 'main.ts') - }, - output: { - path: path.resolve(__dirname, 'webpack-build'), - filename: '[name].js' - }, - optimization: { - minimize: !configuration.debug, - splitChunks: { - cacheGroups: { - styles: { - name: 'cotton_candy', - // Group all CSS files into a single file. - test: /\.css$/, - chunks: 'all', - enforce: true - } - } - } - }, - plugins: [ - new CleanWebpackPlugin(), - new MiniCssExtractPlugin({ - filename: '[name].css' - }) - ], - resolve: { - extensions: ['', '.ts', '.js', '*'] - }, - module: { - rules: [ - { - test: /\.(s?css)$/, - use: [ - { - loader: MiniCssExtractPlugin.loader, - options: { - publicPath: '/' - } - }, - { - loader: 'css-loader', - options: { - url: false - } - }, - { - loader: 'postcss-loader', - options: { - postcssOptions: { - plugins: () => [ - require('autoprefixer') - ] - } - } - }, - { - loader: 'sass-loader' - } - ] - }, - { - test: /\.(js|ts)$/, - exclude: /node_modules/, - use: { - loader: 'babel-loader', - options: { - presets: [ - [ - '@babel/preset-env', { - debug: configuration.debug, - useBuiltIns: 'usage', - corejs: 3 - } - ], - '@babel/preset-typescript' - ], - cacheDirectory: configuration.cacheDirectory - } - } - } - ] - } -} diff --git a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/webpack.config.json.j2 b/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/webpack.config.json.j2 deleted file mode 100644 index 69af3962e..000000000 --- a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/webpack.config.json.j2 +++ /dev/null @@ -1,4 +0,0 @@ -{{ { - 'debug': app.project.configuration.debug, - 'cacheDirectory': app.cache.with_scope(app.extensions['betty.extension.CottonCandy'].name()).with_scope(app.project.name).with_scope('babel').path | str, -} | tojson }} diff --git a/betty/extension/cotton_candy/assets/templates/base.html.j2 b/betty/extension/cotton_candy/assets/templates/base.html.j2 index e3f20b0b3..a07c1640d 100644 --- a/betty/extension/cotton_candy/assets/templates/base.html.j2 +++ b/betty/extension/cotton_candy/assets/templates/base.html.j2 @@ -4,8 +4,6 @@ {% include 'head.html.j2' %} - - + diff --git a/betty/extension/http_api_doc/webpack/main.ts b/betty/extension/http_api_doc/webpack/main.ts new file mode 100644 index 000000000..e69de29bb diff --git a/betty/extension/http_api_doc/assets/betty.extension.npm._Npm/src/package.json b/betty/extension/http_api_doc/webpack/package.json similarity index 100% rename from betty/extension/http_api_doc/assets/betty.extension.npm._Npm/src/package.json rename to betty/extension/http_api_doc/webpack/package.json diff --git a/betty/extension/maps/__init__.py b/betty/extension/maps/__init__.py index 67ec4d0e7..33ce5434a 100644 --- a/betty/extension/maps/__init__.py +++ b/betty/extension/maps/__init__.py @@ -2,89 +2,33 @@ from __future__ import annotations -import asyncio -import logging -from contextlib import suppress +from collections.abc import Sequence from pathlib import Path -from shutil import copy2, copytree - -from aiofiles.os import makedirs from betty.app.extension import Extension, UserFacingExtension -from betty.extension.npm import _Npm, _NpmBuilder, npm, _NpmBuilderCacheScope -from betty.generate import Generator, GenerationContext -from betty.html import CssProvider, JsProvider +from betty.extension.webpack import Webpack, WebpackEntrypointProvider from betty.locale import Str -class Maps(UserFacingExtension, CssProvider, JsProvider, Generator, _NpmBuilder): +class Maps(UserFacingExtension, WebpackEntrypointProvider): @classmethod def name(cls) -> str: return "betty.extension.Maps" @classmethod def depends_on(cls) -> set[type[Extension]]: - return {_Npm} - - async def npm_build( - self, working_directory_path: Path, assets_directory_path: Path - ) -> None: - await self.app.extensions[_Npm].install(type(self), working_directory_path) - await npm(("run", "webpack"), cwd=working_directory_path) - await self._copy_npm_build( - working_directory_path / "webpack-build", assets_directory_path - ) - logging.getLogger(__name__).info( - self._app.localizer._("Built the interactive maps.") - ) - - async def _copy_npm_build( - self, source_directory_path: Path, destination_directory_path: Path - ) -> None: - await makedirs(destination_directory_path, exist_ok=True) - await asyncio.to_thread( - copy2, - source_directory_path / "maps.css", - destination_directory_path / "maps.css", - ) - await asyncio.to_thread( - copy2, - source_directory_path / "maps.js", - destination_directory_path / "maps.js", - ) - with suppress(FileNotFoundError): - await asyncio.to_thread( - copytree, - source_directory_path / "images", - destination_directory_path / "images", - ) - - @classmethod - def npm_cache_scope(cls) -> _NpmBuilderCacheScope: - return _NpmBuilderCacheScope.BETTY - - async def generate(self, job_context: GenerationContext) -> None: - assets_directory_path = await self.app.extensions[_Npm].ensure_assets(self) - await makedirs(self.app.project.configuration.www_directory_path, exist_ok=True) - await self._copy_npm_build( - assets_directory_path, self.app.project.configuration.www_directory_path - ) + return {Webpack} @classmethod - def assets_directory_path(cls) -> Path | None: + def assets_directory_path(cls) -> Path: return Path(__file__).parent / "assets" - @property - def public_css_paths(self) -> list[str]: - return [ - self.app.static_url_generator.generate("maps.css"), - ] + @classmethod + def webpack_entrypoint_directory_path(cls) -> Path: + return Path(__file__).parent / "webpack" - @property - def public_js_paths(self) -> list[str]: - return [ - self.app.static_url_generator.generate("maps.js"), - ] + def webpack_entrypoint_cache_keys(self) -> Sequence[str]: + return () @classmethod def label(cls) -> Str: diff --git a/betty/extension/maps/assets/betty.extension.npm._Npm/.gitignore b/betty/extension/maps/assets/betty.extension.npm._Npm/.gitignore deleted file mode 100644 index 796b96d1c..000000000 --- a/betty/extension/maps/assets/betty.extension.npm._Npm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build diff --git a/betty/extension/maps/assets/betty.extension.npm._Npm/src/.browserslistrc b/betty/extension/maps/assets/betty.extension.npm._Npm/src/.browserslistrc deleted file mode 100644 index 76369daf3..000000000 --- a/betty/extension/maps/assets/betty.extension.npm._Npm/src/.browserslistrc +++ /dev/null @@ -1,3 +0,0 @@ ->0.5% -ie >= 11 -ios >= 9 diff --git a/betty/extension/maps/assets/betty.extension.npm._Npm/src/package.json b/betty/extension/maps/assets/betty.extension.npm._Npm/src/package.json deleted file mode 100644 index 8ef06f05f..000000000 --- a/betty/extension/maps/assets/betty.extension.npm._Npm/src/package.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "main": "maps.js", - "engines": { - "node": ">= 16" - }, - "dependencies": { - "@babel/core": "^7.19.6", - "@babel/preset-env": "^7.19.4", - "babel-loader": "^9.1.0", - "clean-webpack-plugin": "^4.0.0", - "core-js": "^3.26.0", - "css-loader": "^7.1.0", - "leaflet": "^1.9.2", - "leaflet.markercluster": "^1.5.3", - "leaflet.fullscreen": "^3.0.0", - "leaflet-gesture-handling": "^1.2.2", - "mini-css-extract-plugin": "^2.6.1", - "webpack": "^5.74.0", - "webpack-cli": "^5.1.4" - }, - "scripts": { - "webpack": "webpack --config webpack.config.js" - }, - "type": "module" -} diff --git a/betty/extension/maps/assets/betty.extension.npm._Npm/src/webpack.config.js b/betty/extension/maps/assets/betty.extension.npm._Npm/src/webpack.config.js deleted file mode 100644 index 5957a05d6..000000000 --- a/betty/extension/maps/assets/betty.extension.npm._Npm/src/webpack.config.js +++ /dev/null @@ -1,86 +0,0 @@ -'use strict' - -import { CleanWebpackPlugin } from 'clean-webpack-plugin' -import MiniCssExtractPlugin from 'mini-css-extract-plugin' -import path from 'path' -import { readFile } from 'node:fs/promises' -import url from 'node:url' - -const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) -const configuration = JSON.parse(await readFile('./webpack.config.json')) - -export default { - mode: configuration.debug ? 'development' : 'production', - entry: { - maps: path.resolve(__dirname, 'maps.js') - }, - output: { - path: path.resolve(__dirname, 'webpack-build'), - filename: '[name].js' - }, - optimization: { - minimize: !configuration.debug, - splitChunks: { - cacheGroups: { - styles: { - name: 'maps', - // Group all CSS files into a single file. - test: /\.css$/, - chunks: 'all', - enforce: true - } - } - } - }, - plugins: [ - new CleanWebpackPlugin(), - new MiniCssExtractPlugin({ - filename: '[name].css' - }) - ], - module: { - rules: [ - { - test: /\.css$/, - use: [ - { - loader: MiniCssExtractPlugin.loader, - options: { - publicPath: '/' - } - }, - { - loader: 'css-loader' - } - ] - }, - { - test: /\.js$/, - exclude: /node_modules/, - use: { - loader: 'babel-loader', - options: { - presets: [ - [ - '@babel/preset-env', { - debug: configuration.debug, - useBuiltIns: 'usage', - corejs: 3 - } - ] - ], - cacheDirectory: configuration.cacheDirectory - } - } - }, - // Bundle Leaflet images. - { - test: /.*\.png|svg$/, - type: 'asset/resource', - generator: { - filename: 'images/[hash][ext]' - } - } - ] - } -} diff --git a/betty/extension/maps/assets/betty.extension.npm._Npm/src/webpack.config.json.j2 b/betty/extension/maps/assets/betty.extension.npm._Npm/src/webpack.config.json.j2 deleted file mode 100644 index ce9b6cb62..000000000 --- a/betty/extension/maps/assets/betty.extension.npm._Npm/src/webpack.config.json.j2 +++ /dev/null @@ -1,4 +0,0 @@ -{{ { - 'debug': app.project.configuration.debug, - 'cacheDirectory': app.cache.with_scope(app.extensions['betty.extension.Maps'].name()).with_scope(app.project.name).with_scope('babel').path | str, -} | tojson }} \ No newline at end of file diff --git a/betty/extension/maps/assets/betty.extension.npm._Npm/src/maps.css b/betty/extension/maps/webpack/main.css similarity index 100% rename from betty/extension/maps/assets/betty.extension.npm._Npm/src/maps.css rename to betty/extension/maps/webpack/main.css diff --git a/betty/extension/maps/webpack/main.ts b/betty/extension/maps/webpack/main.ts new file mode 100644 index 000000000..ac3b2ffe3 --- /dev/null +++ b/betty/extension/maps/webpack/main.ts @@ -0,0 +1,8 @@ +'use strict' + +import { initializePlaceLists } from './maps.js' + +async function main(): Promise { + await initializePlaceLists() +} +void main() diff --git a/betty/extension/maps/assets/betty.extension.npm._Npm/src/maps.js b/betty/extension/maps/webpack/maps.js similarity index 96% rename from betty/extension/maps/assets/betty.extension.npm._Npm/src/maps.js rename to betty/extension/maps/webpack/maps.js index 40668f179..32fa7b5e3 100644 --- a/betty/extension/maps/assets/betty.extension.npm._Npm/src/maps.js +++ b/betty/extension/maps/webpack/maps.js @@ -1,6 +1,6 @@ 'use strict' -import './maps.css' +import './main.css' import * as L from 'leaflet' import 'leaflet/dist/leaflet.css' @@ -71,4 +71,6 @@ const BettyIcon = L.Icon.Default.extend({ } }) -document.addEventListener('DOMContentLoaded', initializePlaceLists) +export { + initializePlaceLists, +} diff --git a/betty/extension/maps/webpack/package.json b/betty/extension/maps/webpack/package.json new file mode 100644 index 000000000..5b5cef76f --- /dev/null +++ b/betty/extension/maps/webpack/package.json @@ -0,0 +1,11 @@ +{ + "engines": { + "node": ">= 16" + }, + "dependencies": { + "leaflet": "^1.9.2", + "leaflet.markercluster": "^1.5.3", + "leaflet.fullscreen": "^3.0.0", + "leaflet-gesture-handling": "^1.2.2" + } +} diff --git a/betty/extension/npm/__init__.py b/betty/extension/npm/__init__.py deleted file mode 100644 index 7be63adab..000000000 --- a/betty/extension/npm/__init__.py +++ /dev/null @@ -1,281 +0,0 @@ -""" -Provide tools to integrate extensions with `npm `_. - -This extension and module are internal. -""" - -from __future__ import annotations - -import asyncio -import logging -import os -import shutil -import sys -from asyncio import subprocess as aiosubprocess -from contextlib import suppress -from enum import unique, IntFlag, auto -from pathlib import Path -from subprocess import CalledProcessError -from typing import Sequence - -from aiofiles.tempfile import TemporaryDirectory - -from betty.app.extension import Extension, discover_extension_types -from betty.requirement import Requirement, AnyRequirement, AllRequirements -from betty.asyncio import wait_to_thread -from betty.cache.file import BinaryFileCache -from betty.fs import iterfiles -from betty.locale import Str, DEFAULT_LOCALIZER -from betty.subprocess import run_process - - -async def npm( - arguments: Sequence[str], - cwd: Path | None = None, -) -> aiosubprocess.Process: - """ - Run an npm command. - """ - return await run_process( - ["npm", *arguments], - cwd=cwd, - # Use a shell on Windows so subprocess can find the executables it needs (see - # https://bugs.python.org/issue17023). - shell=sys.platform.startswith("win32"), - ) - - -class _NpmRequirement(Requirement): - def __init__(self, met: bool): - super().__init__() - self._met = met - self._summary = self._met_summary() if met else self._unmet_summary() - self._details = Str._( - "npm (https://www.npmjs.com/) must be available for features that require Node.js packages to be installed. Ensure that the `npm` executable is available in your `PATH`." - ) - - @classmethod - def _met_summary(cls) -> Str: - return Str._("`npm` is available") - - @classmethod - def _unmet_summary(cls) -> Str: - return Str._("`npm` is not available") - - @classmethod - def check(cls) -> _NpmRequirement: - try: - wait_to_thread(npm(["--version"])) - logging.getLogger(__name__).debug( - cls._met_summary().localize(DEFAULT_LOCALIZER) - ) - return cls(True) - except (CalledProcessError, FileNotFoundError): - logging.getLogger(__name__).debug( - cls._unmet_summary().localize(DEFAULT_LOCALIZER) - ) - return cls(False) - - def is_met(self) -> bool: - return self._met - - def summary(self) -> Str: - return self._summary - - def details(self) -> Str: - return self._details - - -def is_assets_build_directory_path(path: Path) -> bool: - """ - Check if the given path is an assets build directory path. - """ - return path.is_dir() and len(os.listdir(path)) > 0 - - -class _AssetsRequirement(Requirement): - def __init__(self, extension_types: set[type[_NpmBuilder & Extension]]): - super().__init__() - self._extension_types = extension_types - self._summary = Str._("Pre-built assets") - self._details: Str - if not self.is_met(): - extension_names = sorted( - extension_type.name() - for extension_type in self._extension_types - - self._extension_types_with_built_assets - ) - self._details = Str._( - "Pre-built assets are unavailable for {extension_names}.", - extension_names=", ".join( - extension_names, - ), - ) - else: - self._details = Str.plain("") - - @property - def _extension_types_with_built_assets(self) -> set[type[_NpmBuilder & Extension]]: - return { - extension_type - for extension_type in self._extension_types - if is_assets_build_directory_path( - _get_assets_build_directory_path(extension_type) - ) - } - - def is_met(self) -> bool: - return self._extension_types <= self._extension_types_with_built_assets - - def summary(self) -> Str: - return self._summary - - def details(self) -> Str: - return self._details - - -@unique -class _NpmBuilderCacheScope(IntFlag): - BETTY = auto() - PROJECT = auto() - - -class _NpmBuilder: - async def npm_build( - self, working_directory_path: Path, assets_directory_path: Path - ) -> None: - raise NotImplementedError(repr(self)) - - @classmethod - def npm_cache_scope(cls) -> _NpmBuilderCacheScope: - return _NpmBuilderCacheScope.PROJECT - - -def discover_npm_builders() -> set[type[_NpmBuilder & Extension]]: - """ - Gather all extensions that are npm builders. - """ - return { - extension_type - for extension_type in discover_extension_types() - if issubclass(extension_type, _NpmBuilder) - } - - -def _get_assets_directory_path(extension_type: type[_NpmBuilder & Extension]) -> Path: - assert issubclass(extension_type, Extension) - assert issubclass(extension_type, _NpmBuilder) - assets_directory_path = extension_type.assets_directory_path() - if not assets_directory_path: - raise RuntimeError( - f"Extension {extension_type} does not have an assets directory." - ) - return assets_directory_path / _Npm.name() - - -def _get_assets_src_directory_path( - extension_type: type[_NpmBuilder & Extension], -) -> Path: - return _get_assets_directory_path(extension_type) / "src" - - -def _get_assets_build_directory_path( - extension_type: type[_NpmBuilder & Extension], -) -> Path: - return _get_assets_directory_path(extension_type) / "build" - - -async def build_assets(extension: _NpmBuilder & Extension) -> Path: - """ - Build the npm assets for an extension. - """ - assets_directory_path = _get_assets_build_directory_path(type(extension)) - await _build_assets_to_directory_path(extension, assets_directory_path) - return assets_directory_path - - -async def _build_assets_to_directory_path( - extension: _NpmBuilder & Extension, assets_directory_path: Path -) -> None: - assert isinstance(extension, Extension) - assert isinstance(extension, _NpmBuilder) - with suppress(FileNotFoundError): - await asyncio.to_thread(shutil.rmtree, assets_directory_path) - os.makedirs(assets_directory_path) - async with TemporaryDirectory() as working_directory_path_str: - working_directory_path = Path(working_directory_path_str) - await extension.npm_build(Path(working_directory_path), assets_directory_path) - - -class _Npm(Extension): - _npm_requirement: _NpmRequirement | None = None - _assets_requirement: _AssetsRequirement | None = None - _requirement: Requirement | None = None - - @classmethod - def _ensure_requirement(cls) -> Requirement: - if cls._requirement is None: - cls._npm_requirement = _NpmRequirement.check() - cls._assets_requirement = _AssetsRequirement(discover_npm_builders()) - assert cls._npm_requirement is not None - assert cls._assets_requirement is not None - cls._requirement = AnyRequirement( - cls._npm_requirement, cls._assets_requirement - ) - return cls._requirement - - @classmethod - def enable_requirement(cls) -> Requirement: - return AllRequirements( - cls._ensure_requirement(), - super().enable_requirement(), - ) - - async def install( - self, - extension_type: type[_NpmBuilder & Extension], - working_directory_path: Path, - ) -> None: - self._ensure_requirement() - if self._npm_requirement: - self._npm_requirement.assert_met() - - await asyncio.to_thread( - shutil.copytree, - _get_assets_src_directory_path(extension_type), - working_directory_path, - dirs_exist_ok=True, - ) - async for file_path in iterfiles(working_directory_path): - await self._app.renderer.render_file(file_path) - await npm(["install", "--production"], cwd=working_directory_path) - - def _get_assets_build_cache( - self, extension_type: type[_NpmBuilder & Extension] - ) -> BinaryFileCache: - cache = self._app.binary_file_cache.with_scope(self.name()).with_scope( - extension_type.name() - ) - if extension_type.npm_cache_scope() == _NpmBuilderCacheScope.PROJECT: - cache = cache.with_scope(self.app.project.name) - return cache - - async def ensure_assets(self, extension: _NpmBuilder & Extension) -> Path: - assets_build_directory_paths = [ - _get_assets_build_directory_path(type(extension)), - self._get_assets_build_cache(type(extension)).path, - ] - for assets_build_directory_path in assets_build_directory_paths: - if is_assets_build_directory_path(assets_build_directory_path): - return assets_build_directory_path - - if self._npm_requirement: - self._npm_requirement.assert_met() - return (await self._build_cached_assets(extension)).path - - async def _build_cached_assets( - self, extension: _NpmBuilder & Extension - ) -> BinaryFileCache: - cache = self._get_assets_build_cache(type(extension)) - await _build_assets_to_directory_path(extension, cache.path) - return cache diff --git a/betty/extension/trees/__init__.py b/betty/extension/trees/__init__.py index 2e5a23663..6d805ac56 100644 --- a/betty/extension/trees/__init__.py +++ b/betty/extension/trees/__init__.py @@ -2,81 +2,33 @@ from __future__ import annotations -import asyncio -import logging +from collections.abc import Sequence from pathlib import Path -from shutil import copy2 - -from aiofiles.os import makedirs from betty.app.extension import Extension, UserFacingExtension -from betty.extension.npm import _Npm, _NpmBuilder, npm, _NpmBuilderCacheScope -from betty.generate import Generator, GenerationContext -from betty.html import CssProvider, JsProvider +from betty.extension.webpack import Webpack, WebpackEntrypointProvider from betty.locale import Str -class Trees(UserFacingExtension, CssProvider, JsProvider, Generator, _NpmBuilder): +class Trees(UserFacingExtension, WebpackEntrypointProvider): @classmethod def name(cls) -> str: return "betty.extension.Trees" @classmethod def depends_on(cls) -> set[type[Extension]]: - return {_Npm} - - async def npm_build( - self, working_directory_path: Path, assets_directory_path: Path - ) -> None: - await self.app.extensions[_Npm].install(type(self), working_directory_path) - await npm(("run", "webpack"), cwd=working_directory_path) - await self._copy_npm_build( - working_directory_path / "webpack-build", assets_directory_path - ) - logging.getLogger(__name__).info( - self._app.localizer._("Built the interactive family trees.") - ) - - async def _copy_npm_build( - self, source_directory_path: Path, destination_directory_path: Path - ) -> None: - await makedirs(destination_directory_path, exist_ok=True) - await asyncio.to_thread( - copy2, - source_directory_path / "trees.css", - destination_directory_path / "trees.css", - ) - await asyncio.to_thread( - copy2, - source_directory_path / "trees.js", - destination_directory_path / "trees.js", - ) - - @classmethod - def npm_cache_scope(cls) -> _NpmBuilderCacheScope: - return _NpmBuilderCacheScope.BETTY - - async def generate(self, job_context: GenerationContext) -> None: - assets_directory_path = await self.app.extensions[_Npm].ensure_assets(self) - await self._copy_npm_build( - assets_directory_path, self.app.project.configuration.www_directory_path - ) + return {Webpack} @classmethod - def assets_directory_path(cls) -> Path | None: + def assets_directory_path(cls) -> Path: return Path(__file__).parent / "assets" - @property - def public_css_paths(self) -> list[str]: - return [ - self.app.static_url_generator.generate("trees.css"), - ] + @classmethod + def webpack_entrypoint_directory_path(cls) -> Path: + return Path(__file__).parent / "webpack" - @property - def public_js_paths(self) -> list[str]: - return [ - self.app.static_url_generator.generate("trees.js"), - ] + def webpack_entrypoint_cache_keys(self) -> Sequence[str]: + return () @classmethod def label(cls) -> Str: diff --git a/betty/extension/trees/assets/betty.extension.npm._Npm/.gitignore b/betty/extension/trees/assets/betty.extension.npm._Npm/.gitignore deleted file mode 100644 index 796b96d1c..000000000 --- a/betty/extension/trees/assets/betty.extension.npm._Npm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/build diff --git a/betty/extension/trees/assets/betty.extension.npm._Npm/src/.browserslistrc b/betty/extension/trees/assets/betty.extension.npm._Npm/src/.browserslistrc deleted file mode 100644 index 76369daf3..000000000 --- a/betty/extension/trees/assets/betty.extension.npm._Npm/src/.browserslistrc +++ /dev/null @@ -1,3 +0,0 @@ ->0.5% -ie >= 11 -ios >= 9 diff --git a/betty/extension/trees/assets/betty.extension.npm._Npm/src/package.json b/betty/extension/trees/assets/betty.extension.npm._Npm/src/package.json deleted file mode 100644 index adf930df8..000000000 --- a/betty/extension/trees/assets/betty.extension.npm._Npm/src/package.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "main": "trees.js", - "engines": { - "node": ">= 16" - }, - "dependencies": { - "@babel/core": "^7.19.6", - "@babel/preset-env": "^7.19.4", - "babel-loader": "^9.1.0", - "clean-webpack-plugin": "^4.0.0", - "core-js": "^3.26.0", - "css-loader": "^7.1.0", - "cytoscape": "^3.23.0", - "cytoscape-dagre": "^2.5.0", - "mini-css-extract-plugin": "^2.6.1", - "webpack": "^5.74.0", - "webpack-cli": "^5.1.4" - }, - "scripts": { - "webpack": "webpack --config webpack.config.js" - }, - "type": "module" -} diff --git a/betty/extension/trees/assets/betty.extension.npm._Npm/src/webpack.config.js b/betty/extension/trees/assets/betty.extension.npm._Npm/src/webpack.config.js deleted file mode 100644 index c63413f1e..000000000 --- a/betty/extension/trees/assets/betty.extension.npm._Npm/src/webpack.config.js +++ /dev/null @@ -1,78 +0,0 @@ -'use strict' - -import { CleanWebpackPlugin } from 'clean-webpack-plugin' -import MiniCssExtractPlugin from 'mini-css-extract-plugin' -import path from 'path' -import { readFile } from 'node:fs/promises' -import url from 'node:url' - -const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) -const configuration = JSON.parse(await readFile('./webpack.config.json')) - -export default { - mode: configuration.debug ? 'development' : 'production', - entry: { - trees: path.resolve(__dirname, 'trees.js') - }, - output: { - path: path.resolve(__dirname, 'webpack-build'), - filename: '[name].js' - }, - optimization: { - minimize: !configuration.debug, - splitChunks: { - cacheGroups: { - styles: { - name: 'trees', - // Group all CSS files into a single file. - test: /\.css$/, - chunks: 'all', - enforce: true - } - } - } - }, - plugins: [ - new CleanWebpackPlugin(), - new MiniCssExtractPlugin({ - filename: '[name].css' - }) - ], - module: { - rules: [ - { - test: /\.css$/, - use: [ - { - loader: MiniCssExtractPlugin.loader, - options: { - publicPath: '/' - } - }, - { - loader: 'css-loader' - } - ] - }, - { - test: /\.js$/, - exclude: /node_modules/, - use: { - loader: 'babel-loader', - options: { - presets: [ - [ - '@babel/preset-env', { - debug: configuration.debug, - useBuiltIns: 'usage', - corejs: 3 - } - ] - ], - cacheDirectory: configuration.cacheDirectory - } - } - } - ] - } -} diff --git a/betty/extension/trees/assets/betty.extension.npm._Npm/src/webpack.config.json.j2 b/betty/extension/trees/assets/betty.extension.npm._Npm/src/webpack.config.json.j2 deleted file mode 100644 index c6987db8d..000000000 --- a/betty/extension/trees/assets/betty.extension.npm._Npm/src/webpack.config.json.j2 +++ /dev/null @@ -1,4 +0,0 @@ -{{ { - 'debug': app.project.configuration.debug, - 'cacheDirectory': app.cache.with_scope(app.extensions['betty.extension.Trees'].name()).with_scope(app.project.name).with_scope('babel').path | str, -} | tojson }} diff --git a/betty/extension/trees/assets/betty.extension.npm._Npm/src/trees.css b/betty/extension/trees/webpack/main.css similarity index 100% rename from betty/extension/trees/assets/betty.extension.npm._Npm/src/trees.css rename to betty/extension/trees/webpack/main.css diff --git a/betty/extension/trees/webpack/main.ts b/betty/extension/trees/webpack/main.ts new file mode 100644 index 000000000..f92818854 --- /dev/null +++ b/betty/extension/trees/webpack/main.ts @@ -0,0 +1,8 @@ +'use strict' + +import { initializeAncestryTrees } from './trees.js' + +async function main(): Promise { + await initializeAncestryTrees() +} +void main() diff --git a/betty/extension/trees/webpack/package.json b/betty/extension/trees/webpack/package.json new file mode 100644 index 000000000..d405878ef --- /dev/null +++ b/betty/extension/trees/webpack/package.json @@ -0,0 +1,9 @@ +{ + "engines": { + "node": ">= 16" + }, + "dependencies": { + "cytoscape": "^3.23.0", + "cytoscape-dagre": "^2.5.0" + } +} diff --git a/betty/extension/trees/assets/betty.extension.npm._Npm/src/trees.js b/betty/extension/trees/webpack/trees.js similarity index 97% rename from betty/extension/trees/assets/betty.extension.npm._Npm/src/trees.js rename to betty/extension/trees/webpack/trees.js index 8aaba7cea..7c45713da 100644 --- a/betty/extension/trees/assets/betty.extension.npm._Npm/src/trees.js +++ b/betty/extension/trees/webpack/trees.js @@ -1,6 +1,6 @@ 'use strict' -import './trees.css' +import './main.css' import cytoscape from 'cytoscape' import dagre from 'cytoscape-dagre' @@ -126,4 +126,6 @@ function childrenToElements (parent, elements, people) { } } -document.addEventListener('DOMContentLoaded', initializeAncestryTrees) +export { + initializeAncestryTrees, +} diff --git a/betty/extension/webpack/__init__.py b/betty/extension/webpack/__init__.py new file mode 100644 index 000000000..b4155eea2 --- /dev/null +++ b/betty/extension/webpack/__init__.py @@ -0,0 +1,195 @@ +""" +Integrate Betty with `Webpack `_. + +This module is internal. +""" + +from __future__ import annotations + +from asyncio import to_thread +from collections.abc import Callable, Sequence +from pathlib import Path +from shutil import copytree +from typing import Any + +from aiofiles.tempfile import TemporaryDirectory + +from betty import fs +from betty._npm import NpmRequirement, NpmUnavailable +from betty.app.extension import Extension +from betty.extension.webpack import build +from betty.extension.webpack.build import webpack_build_id +from betty.extension.webpack.jinja2.filter import FILTERS +from betty.generate import Generator, GenerationContext +from betty.html import CssProvider +from betty.jinja2 import Jinja2Provider +from betty.job import Context +from betty.locale import Str +from betty.requirement import ( + Requirement, + AllRequirements, + AnyRequirement, + RequirementError, +) + + +def _prebuilt_webpack_build_directory_path( + entrypoint_providers: Sequence[WebpackEntrypointProvider & Extension], +) -> Path: + return ( + fs.PREBUILT_ASSETS_DIRECTORY_PATH + / "webpack" + / f"build-{webpack_build_id(entrypoint_providers)}" + ) + + +class WebpackEntrypointProvider: + @classmethod + def webpack_entrypoint_directory_path(cls) -> Path: + """ + Get the path to the directory with the entrypoint assets. + + The directory must include at least a ``package.json`` and ``main.ts``. + """ + raise NotImplementedError + + def webpack_entrypoint_cache_keys(self) -> Sequence[str]: + """ + Get the keys that make a Webpack build for this provider unique. + + Providers that can be cached regardless may ``return ()``. + """ + raise NotImplementedError + + +class PrebuiltAssetsRequirement(Requirement): + def is_met(self) -> bool: + return (fs.PREBUILT_ASSETS_DIRECTORY_PATH / "webpack").is_dir() + + def summary(self) -> Str: + return ( + Str._("Pre-built Webpack front-end assets are available") + if self.is_met() + else Str._("Pre-built Webpack front-end assets are unavailable") + ) + + +class Webpack(Extension, CssProvider, Jinja2Provider, Generator): + _npm_requirement = NpmRequirement() + _prebuilt_assets_requirement = PrebuiltAssetsRequirement() + _requirement = AnyRequirement( + _npm_requirement, + _prebuilt_assets_requirement, + ) + + @classmethod + def name(cls) -> str: + return "betty.extension.Webpack" + + @classmethod + def enable_requirement(cls) -> Requirement: + return AllRequirements(super().enable_requirement(), cls._requirement) + + def build_requirement(self) -> Requirement: + return self._npm_requirement + + @classmethod + def assets_directory_path(cls) -> Path: + return Path(__file__).parent / "assets" + + @property + def public_css_paths(self) -> list[str]: + return [ + self.app.static_url_generator.generate("css/vendor.css"), + ] + + def new_context_vars(self) -> dict[str, Any]: + return { + "webpack_js_entrypoints": set(), + } + + @property + def filters(self) -> dict[str, Callable[..., Any]]: + return FILTERS + + @property + def _project_entrypoint_providers( + self, + ) -> Sequence[WebpackEntrypointProvider & Extension]: + return [ + extension + for extension in self._app.extensions.flatten() + if isinstance(extension, WebpackEntrypointProvider) + ] + + async def generate(self, job_context: GenerationContext) -> None: + build_directory_path = await self._generate_ensure_build_directory( + job_context=job_context, + ) + await self._copy_build_directory( + build_directory_path, self._app.project.configuration.www_directory_path + ) + + async def prebuild(self, job_context: Context) -> None: + async with TemporaryDirectory() as working_directory_path_str: + build_directory_path = await self._new_builder( + Path(working_directory_path_str), + job_context=job_context, + ).build() + await self._copy_build_directory( + build_directory_path, + _prebuilt_webpack_build_directory_path( + self._project_entrypoint_providers + ), + ) + + def _new_builder( + self, + working_directory_path: Path, + *, + job_context: Context, + ) -> build.Builder: + return build.Builder( + working_directory_path, + self._project_entrypoint_providers, + self._app.project.configuration.debug, + self._app.renderer, + job_context=job_context, + localizer=self._app.localizer, + ) + + async def _copy_build_directory( + self, + build_directory_path: Path, + destination_directory_path: Path, + ) -> None: + await to_thread( + copytree, + build_directory_path, + destination_directory_path, + dirs_exist_ok=True, + ) + + async def _generate_ensure_build_directory( + self, + *, + job_context: Context, + ) -> Path: + builder = self._new_builder( + self._app.binary_file_cache.with_scope("webpack").path, + job_context=job_context, + ) + try: + # (Re)build the assets if `npm` is available. + return await builder.build() + except NpmUnavailable: + pass + + # Use prebuilt assets if they exist. + prebuilt_webpack_build_directory_path = _prebuilt_webpack_build_directory_path( + self._project_entrypoint_providers + ) + if prebuilt_webpack_build_directory_path.exists(): + return prebuilt_webpack_build_directory_path + + raise RequirementError(self._requirement) diff --git a/betty/extension/webpack/assets/templates/webpack-entry-loader.html.j2 b/betty/extension/webpack/assets/templates/webpack-entry-loader.html.j2 new file mode 100644 index 000000000..85d789af2 --- /dev/null +++ b/betty/extension/webpack/assets/templates/webpack-entry-loader.html.j2 @@ -0,0 +1 @@ + diff --git a/betty/extension/webpack/build.py b/betty/extension/webpack/build.py new file mode 100644 index 000000000..29fda38b0 --- /dev/null +++ b/betty/extension/webpack/build.py @@ -0,0 +1,257 @@ +""" +Perform Webpack builds. +""" + +from __future__ import annotations + +from asyncio import to_thread +from collections.abc import Sequence, MutableMapping +from json import dumps, loads +from logging import getLogger +from pathlib import Path +from shutil import copy2 +from typing import TYPE_CHECKING + +import aiofiles +from aiofiles.os import makedirs + +from betty import _npm +from betty.app.extension import Extension +from betty.asyncio import gather +from betty.fs import ROOT_DIRECTORY_PATH, iterfiles +from betty.hashid import hashid, hashid_sequence, hashid_file_content +from betty.job import Context +from betty.locale import Localizer +from betty.render import Renderer + +if TYPE_CHECKING: + from betty.extension.webpack import WebpackEntrypointProvider + + +_NPM_PROJECT_DIRECTORIES_PATH = Path(__file__).parent / "webpack" + + +async def _npm_project_id( + entrypoint_providers: Sequence[WebpackEntrypointProvider & Extension], debug: bool +) -> str: + return hashid_sequence( + "true" if debug else "false", + await hashid_file_content(_NPM_PROJECT_DIRECTORIES_PATH / "package.json"), + *[ + await hashid_file_content( + entrypoint_provider.webpack_entrypoint_directory_path() / "package.json" + ) + for entrypoint_provider in entrypoint_providers + ], + ) + + +async def _npm_project_directory_path( + working_directory_path: Path, + entrypoint_providers: Sequence[WebpackEntrypointProvider & Extension], + debug: bool, +) -> Path: + return working_directory_path / await _npm_project_id(entrypoint_providers, debug) + + +def webpack_build_id( + entrypoint_providers: Sequence[WebpackEntrypointProvider & Extension], +) -> str: + """ + Generate the ID for a Webpack build. + """ + return hashid_sequence( + *( + "-".join( + map( + hashid, + entrypoint_provider.webpack_entrypoint_cache_keys(), + ) + ) + for entrypoint_provider in entrypoint_providers + ) + ) + + +def _webpack_build_directory_path( + npm_project_directory_path: Path, + entrypoint_providers: Sequence[WebpackEntrypointProvider & Extension], +) -> Path: + return ( + npm_project_directory_path / f"build-{webpack_build_id(entrypoint_providers)}" + ) + + +class Builder: + def __init__( + self, + working_directory_path: Path, + entrypoint_providers: Sequence[WebpackEntrypointProvider & Extension], + debug: bool, + renderer: Renderer, + *, + job_context: Context, + localizer: Localizer, + ) -> None: + self._working_directory_path = working_directory_path + self._entrypoint_providers = entrypoint_providers + self._debug = debug + self._renderer = renderer + self._job_context = job_context + self._localizer = localizer + + async def _copy2_and_render( + self, source_path: Path, destination_path: Path + ) -> None: + await makedirs(destination_path.parent, exist_ok=True) + await to_thread(copy2, source_path, destination_path) + await self._renderer.render_file( + source_path, + job_context=self._job_context, + localizer=self._localizer, + ) + + async def _copytree_and_render( + self, source_path: Path, destination_path: Path + ) -> None: + await gather( + *[ + self._copy2_and_render( + file_source_path, + destination_path / file_source_path.relative_to(source_path), + ) + async for file_source_path in iterfiles(source_path) + ] + ) + + async def _prepare_webpack_extension( + self, npm_project_directory_path: Path + ) -> None: + await gather( + *[ + to_thread( + copy2, + source_file_path, + npm_project_directory_path, + ) + for source_file_path in ( + _NPM_PROJECT_DIRECTORIES_PATH / "package.json", + _NPM_PROJECT_DIRECTORIES_PATH / "webpack.config.js", + ROOT_DIRECTORY_PATH / ".browserslistrc", + ROOT_DIRECTORY_PATH / "tsconfig.json", + ) + ] + ) + + async def _prepare_webpack_entrypoint_provider( + self, + npm_project_directory_path: Path, + entrypoint_provider: type[WebpackEntrypointProvider & Extension], + npm_project_package_json_dependencies: MutableMapping[str, str], + webpack_entry: MutableMapping[str, str], + ) -> None: + entrypoint_provider_working_directory_path = ( + npm_project_directory_path / "entrypoints" / entrypoint_provider.name() + ) + await self._copytree_and_render( + entrypoint_provider.webpack_entrypoint_directory_path(), + entrypoint_provider_working_directory_path, + ) + npm_project_package_json_dependencies[entrypoint_provider.name()] = ( + # Ensure a relative path inside the npm project directory, or else npm + # will not install our entrypoints' dependencies. + f"file:{entrypoint_provider_working_directory_path.relative_to(npm_project_directory_path)}" + ) + # Webpack requires relative paths to start with a leading dot and use forward slashes. + webpack_entry[entrypoint_provider.name()] = "/".join( + ( + ".", + *(entrypoint_provider_working_directory_path / "main.ts") + .relative_to(npm_project_directory_path) + .parts, + ) + ) + + async def _prepare_npm_project_directory( + self, npm_project_directory_path: Path, webpack_build_directory_path: Path + ) -> None: + npm_project_package_json_dependencies: MutableMapping[str, str] = {} + webpack_entry: MutableMapping[str, str] = {} + await makedirs(npm_project_directory_path, exist_ok=True) + await gather( + self._prepare_webpack_extension(npm_project_directory_path), + *( + self._prepare_webpack_entrypoint_provider( + npm_project_directory_path, + type(entrypoint_provider), + npm_project_package_json_dependencies, + webpack_entry, + ) + for entrypoint_provider in self._entrypoint_providers + ), + ) + webpack_configuration_json = dumps( + { + # Use a relative path so we avoid portability issues with + # leading root slashes or drive letters. + "buildDirectoryPath": str( + webpack_build_directory_path.relative_to(npm_project_directory_path) + ), + "debug": self._debug, + "entry": webpack_entry, + } + ) + async with aiofiles.open( + npm_project_directory_path / "webpack.config.json", "w" + ) as configuration_f: + await configuration_f.write(webpack_configuration_json) + + # Add dependencies to package.json. + npm_project_package_json_path = npm_project_directory_path / "package.json" + async with aiofiles.open( + npm_project_package_json_path, "r" + ) as npm_project_package_json_f: + npm_project_package_json = loads(await npm_project_package_json_f.read()) + npm_project_package_json["dependencies"].update( + npm_project_package_json_dependencies + ) + async with aiofiles.open( + npm_project_package_json_path, "w" + ) as npm_project_package_json_f: + await npm_project_package_json_f.write(dumps(npm_project_package_json)) + + async def _npm_install(self, npm_project_directory_path: Path) -> None: + await _npm.npm(("install", "--production"), cwd=npm_project_directory_path) + + async def _webpack_build( + self, npm_project_directory_path: Path, webpack_build_directory_path: Path + ) -> None: + await _npm.npm(("run", "webpack"), cwd=npm_project_directory_path) + + # Ensure there is always a vendor.css. This makes for easy and unconditional importing. + await makedirs(webpack_build_directory_path / "css", exist_ok=True) + await to_thread((webpack_build_directory_path / "css" / "vendor.css").touch) + + async def build(self) -> Path: + npm_project_directory_path = await _npm_project_directory_path( + self._working_directory_path, self._entrypoint_providers, self._debug + ) + webpack_build_directory_path = _webpack_build_directory_path( + npm_project_directory_path, + self._entrypoint_providers, + ) + if webpack_build_directory_path.exists(): + return webpack_build_directory_path + npm_install_required = not npm_project_directory_path.exists() + await self._prepare_npm_project_directory( + npm_project_directory_path, webpack_build_directory_path + ) + if npm_install_required: + await self._npm_install(npm_project_directory_path) + await self._webpack_build( + npm_project_directory_path, webpack_build_directory_path + ) + getLogger(__name__).info( + self._localizer._("Built the Webpack front-end assets.") + ) + return webpack_build_directory_path diff --git a/betty/extension/webpack/jinja2/__init__.py b/betty/extension/webpack/jinja2/__init__.py new file mode 100644 index 000000000..971037209 --- /dev/null +++ b/betty/extension/webpack/jinja2/__init__.py @@ -0,0 +1,14 @@ +""" +Integrate Webpack with Jinja2. +""" + +from jinja2.runtime import Context + + +def _context_js_entrypoints(context: Context) -> set[str]: + entrypoints = context.resolve_or_missing("webpack_js_entrypoints") + if isinstance(entrypoints, set): + return entrypoints + raise RuntimeError( + "No `webpack_js_entrypoints` context variable exists in this Jinja2 template." + ) diff --git a/betty/extension/webpack/jinja2/filter.py b/betty/extension/webpack/jinja2/filter.py new file mode 100644 index 000000000..b9b14b064 --- /dev/null +++ b/betty/extension/webpack/jinja2/filter.py @@ -0,0 +1,25 @@ +""" +Provide Jinja2 filters to integrate with Webpack. +""" + +from __future__ import annotations + +from jinja2 import pass_context +from jinja2.runtime import Context + +from betty.extension.webpack.jinja2 import _context_js_entrypoints +from betty.jinja2.filter import filter_public_js + + +@pass_context +def filter_webpack_entrypoint_js(context: Context, entrypoint_name: str) -> None: + """ + Add a Webpack entrypoint's JavaScript files to the current page. + """ + filter_public_js(context, "/js/webpack-entry-loader.js") + _context_js_entrypoints(context).add(entrypoint_name) + + +FILTERS = { + "webpack_entrypoint_js": filter_webpack_entrypoint_js, +} diff --git a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/package.json b/betty/extension/webpack/webpack/package.json similarity index 73% rename from betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/package.json rename to betty/extension/webpack/webpack/package.json index 3d7cca249..fb0968e63 100644 --- a/betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/package.json +++ b/betty/extension/webpack/webpack/package.json @@ -1,5 +1,4 @@ { - "main": "main.ts", "engines": { "node": ">= 16" }, @@ -9,13 +8,19 @@ "@babel/preset-typescript": "^7.24.1", "babel-loader": "^9.1.0", "clean-webpack-plugin": "^4.0.0", + "copy-webpack-plugin": "^12.0.2", "core-js": "^3.26.0", "css-loader": "^7.1.0", + "css-minimizer-webpack-plugin": "^6.0.0", + "file-loader": "^6.2.0", "mini-css-extract-plugin": "^2.6.1", "postcss-loader": "^8.0.0", + "resolve-url-loader": "^5.0.0", "sass": "^1.56.1", "sass-loader": "^14.0.0", "style-loader": "^4.0.0", + "terser-webpack-plugin": "^5.3.10", + "typescript": "^5.4.5", "webpack": "^5.74.0", "webpack-cli": "^5.1.4" }, diff --git a/betty/extension/webpack/webpack/webpack.config.js b/betty/extension/webpack/webpack/webpack.config.js new file mode 100644 index 000000000..2e7ffb0f0 --- /dev/null +++ b/betty/extension/webpack/webpack/webpack.config.js @@ -0,0 +1,199 @@ +'use strict' + +import { CleanWebpackPlugin } from 'clean-webpack-plugin' +import CopyWebpackPlugin from 'copy-webpack-plugin' +import CssMinimizerPlugin from 'css-minimizer-webpack-plugin' +import MiniCssExtractPlugin from 'mini-css-extract-plugin' +import path from 'path' +import { readFile } from 'node:fs/promises' +import TerserPlugin from 'terser-webpack-plugin' +import url from 'node:url' +import webpack from 'webpack' + +const __dirname = url.fileURLToPath(new URL('.', import.meta.url)) +const configuration = JSON.parse(await readFile('./webpack.config.json')) + +/** + * Collect the scripts needed for all entrypoints, and build a loader. + * + * We do this, because page generation and the Webpack build take place concurrently for performance reasons. + * When rendered, pages declare the Webpack extension entrypoints they need. + * Using this Webpack plugin, we build a map of all scripts needed per entrypoint, + * as well as a loader that is run on each page. The loader then imports the + * scripts needed for the entrypoints declared on the page. + */ +class EntryScriptCollector { + apply (compiler) { + compiler.hooks.initialize.tap('ChunkCollector', () => { + compiler.hooks.thisCompilation.tap( + 'ChunkCollector', + (compilation) => { + compilation.hooks.processAssets.tapAsync( + { + name: 'ChunkCollector', + stage: compiler.webpack.Compilation.PROCESS_ASSETS_STAGE_OPTIMIZE_INLINE + }, + (_, callback) => { + const extensionRegexp = /\.(js|mjs)(\?|$)/ + const scripts = {} + for (const entryName of Object.keys(configuration.entry)) { + scripts[entryName] = compilation.entrypoints + .get(entryName) + .getFiles() + .filter(entryFile => extensionRegexp.test(entryFile)) + .map(entryFile => `/${entryFile}`) + } + + const webpackEntryLoader = ` +(async () => { + const entryScriptsMap = ${JSON.stringify(scripts)} + const entryNames = document.getElementById('webpack-entry-loader').dataset.webpackEntryLoader.split(':') + const entryScripts = new Set(entryNames.reduce( + (accumulatedEntryScripts, entryName) => [...accumulatedEntryScripts, ...entryScriptsMap[entryName]], + [], + )) + await Promise.allSettled([...entryScripts].map(async (entryScript) => await import(entryScript))) +})() +` + compilation.emitAsset( + path.join('js', 'webpack-entry-loader.js'), + new webpack.sources.RawSource(webpackEntryLoader) + ) + return callback() + } + ) + } + ) + }) + } +} + +const webpackConfiguration = { + mode: configuration.debug ? 'development' : 'production', + devtool: configuration.debug ? 'eval-source-map' : false, + entry: configuration.entry, + output: { + path: path.resolve(__dirname, configuration.buildDirectoryPath), + filename: 'js/[name].js' + }, + optimization: { + concatenateModules: true, + minimize: !configuration.debug, + minimizer: [ + new CssMinimizerPlugin(), + new TerserPlugin({ + extractComments: false, + terserOptions: { + output: { + comments: false + } + } + }) + ], + splitChunks: { + chunks: 'all', + cacheGroups: { + // The resulting CSS files are one per entrypoint, and a single vendor.css. + // This makes for easy and unconditional importing. + vendorCss: { + test: /[\\/]node_modules[\\/].+?\.css$/, + name: 'vendor', + priority: -10 + }, + vendorJs: { + test: /[\\/]node_modules[\\/].+?\.js$/, + priority: -10 + } + } + }, + runtimeChunk: 'single' + }, + plugins: [ + new CleanWebpackPlugin(), + new CopyWebpackPlugin({ + patterns: [ + // The HttpApiDoc extension does not have a Webpack build as such (yet), but simply + // requires a single dependency distribution asset verbatim. + { + from: path.join('node_modules','redoc', 'bundles', 'redoc.standalone.js'), + to: 'js/http-api-doc.js', + noErrorOnMissing: true, + }, + ], + }), + new EntryScriptCollector(), + new MiniCssExtractPlugin({ + filename: 'css/[name].css' + }) + ], + module: { + rules: [ + { + test: /\.(js|ts)$/, + exclude: /node_modules/, + use: [ + { + loader: 'babel-loader', + options: { + cacheDirectory: path.resolve(__dirname, 'cache'), + presets: [ + [ + '@babel/preset-env', { + debug: configuration.debug, + modules: false, + useBuiltIns: 'usage', + corejs: 3 + }, + ], + '@babel/preset-typescript', + ] + } + } + ] + }, + { + test: /\.s?css$/, + use: [ + { + loader: MiniCssExtractPlugin.loader, + options: { + publicPath: '/' + } + }, + { + loader: 'css-loader', + options: { + url: { + // Betty's own assets are generated through the assets file system, + // so we use Webpack for vendor assets only. + filter: (url, resourcePath) => resourcePath.includes('/node_modules/'), + } + } + }, + { + loader: 'postcss-loader', + options: { + postcssOptions: { + plugins: () => [ + require('autoprefixer') + ] + } + } + }, + { + loader: 'sass-loader' + } + ] + }, + { + test: /.*\.png|gif|jpg|jpeg|svg/, + type: 'asset/resource', + generator: { + filename: 'images/[hash][ext]' + } + } + ] + } +} + +export default webpackConfiguration diff --git a/betty/fs.py b/betty/fs.py index 22a9da4cb..9c705dd79 100644 --- a/betty/fs.py +++ b/betty/fs.py @@ -28,6 +28,9 @@ ASSETS_DIRECTORY_PATH = ROOT_DIRECTORY_PATH / "betty" / "assets" +PREBUILT_ASSETS_DIRECTORY_PATH = ROOT_DIRECTORY_PATH / "prebuild" + + HOME_DIRECTORY_PATH = Path.home() / ".betty" diff --git a/betty/jinja2/__init__.py b/betty/jinja2/__init__.py index cccebe8b0..7568ce833 100644 --- a/betty/jinja2/__init__.py +++ b/betty/jinja2/__init__.py @@ -6,7 +6,7 @@ import datetime from collections import defaultdict -from collections.abc import MutableMapping, Iterator +from collections.abc import MutableMapping, Iterator, Sequence from pathlib import Path from threading import Lock from typing import Callable, Any, cast, TypeVar @@ -22,6 +22,7 @@ from jinja2.runtime import StrictUndefined, Context, DebugUndefined from betty.app import App +from betty.app.extension import Extension from betty.html import CssProvider, JsProvider from betty.jinja2.filter import FILTERS from betty.jinja2.test import TESTS @@ -154,6 +155,9 @@ def filters(self) -> dict[str, Callable[..., Any]]: def tests(self) -> dict[str, Callable[..., bool]]: return {} + def new_context_vars(self) -> dict[str, Any]: + return {} + class Environment(Jinja2Environment): globals: dict[str, Any] @@ -204,6 +208,11 @@ def _init_i18n(self) -> None: @property def context_class(self) -> type[Context]: # type: ignore[override] if self._context_class is None: + jinja2_providers: Sequence[Jinja2Provider & Extension] = [ + extension + for extension in self.app.extensions.flatten() + if isinstance(extension, Jinja2Provider) + ] class _Context(Context): def __init__( @@ -218,6 +227,10 @@ def __init__( parent["citer"] = _Citer() if "breadcrumbs" not in parent: parent["breadcrumbs"] = _Breadcrumbs() + for jinja2_provider in jinja2_providers: + for key, value in jinja2_provider.new_context_vars().items(): + if key not in parent: + parent[key] = value super().__init__( environment, parent, diff --git a/betty/jinja2/filter.py b/betty/jinja2/filter.py index 17eeb8ab6..d9d6fc912 100644 --- a/betty/jinja2/filter.py +++ b/betty/jinja2/filter.py @@ -539,6 +539,28 @@ def filter_hashid(input: str) -> str: return hashid(input) +@pass_context +def filter_public_css(context: Context, public_path: str) -> None: + """ + Add a CSS file to the current page. + """ + public_css_paths = context.resolve_or_missing("public_css_paths") + if public_path in public_css_paths: + return + public_css_paths.append(public_path) + + +@pass_context +def filter_public_js(context: Context, public_path: str) -> None: + """ + Add a JavaScript file to the current page. + """ + public_js_paths = context.resolve_or_missing("public_js_paths") + if public_path in public_js_paths: + return + public_js_paths.append(public_path) + + FILTERS = { "base64": filter_base64, "camel_case_to_kebab_case": camel_case_to_kebab_case, @@ -569,4 +591,6 @@ def filter_hashid(input: str) -> str: "url": filter_url, "void_none": void_none, "walk": filter_walk, + "public_css": filter_public_css, + "public_js": filter_public_js, } diff --git a/betty/tests/conftest.py b/betty/tests/conftest.py index b07b87f0f..f67799d0b 100644 --- a/betty/tests/conftest.py +++ b/betty/tests/conftest.py @@ -5,9 +5,9 @@ from __future__ import annotations import logging -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Iterator from pathlib import Path -from typing import Iterator, TypeVar, cast +from typing import TypeVar, cast from warnings import filterwarnings import pytest diff --git a/betty/tests/extension/http_api_doc/test___init__.py b/betty/tests/extension/http_api_doc/test___init__.py index 1645c0201..1ad3aac1f 100644 --- a/betty/tests/extension/http_api_doc/test___init__.py +++ b/betty/tests/extension/http_api_doc/test___init__.py @@ -15,5 +15,5 @@ async def test_generate(self) -> None: app.project.configuration.www_directory_path / "api" / "index.html" ).is_file() assert ( - app.project.configuration.www_directory_path / "http-api-doc.js" + app.project.configuration.www_directory_path / "js" / "http-api-doc.js" ).is_file() diff --git a/betty/tests/extension/maps/test___init__.py b/betty/tests/extension/maps/test___init__.py index 41c1539e4..f5d6629ac 100644 --- a/betty/tests/extension/maps/test___init__.py +++ b/betty/tests/extension/maps/test___init__.py @@ -13,14 +13,18 @@ async def test_generate(self) -> None: app.project.configuration.extensions.append(ExtensionConfiguration(Maps)) await generate(app) async with aiofiles.open( - app.project.configuration.www_directory_path / "maps.js", + app.project.configuration.www_directory_path + / "js" + / "betty.extension.Maps.js", encoding="utf-8", ) as f: betty_js = await f.read() - assert "maps.js" in betty_js + assert Maps.name() in betty_js async with aiofiles.open( - app.project.configuration.www_directory_path / "maps.css", + app.project.configuration.www_directory_path + / "css" + / "betty.extension.Maps.css", encoding="utf-8", ) as f: betty_css = await f.read() - assert ".map" in betty_css + assert Maps.name() in betty_css diff --git a/betty/tests/extension/npm/__init__.py b/betty/tests/extension/npm/__init__.py deleted file mode 100644 index 8cdae1086..000000000 --- a/betty/tests/extension/npm/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Test the betty.extension.npm module.""" diff --git a/betty/tests/extension/npm/test___init__.py b/betty/tests/extension/npm/test___init__.py deleted file mode 100644 index d5cca56b2..000000000 --- a/betty/tests/extension/npm/test___init__.py +++ /dev/null @@ -1,25 +0,0 @@ -from subprocess import CalledProcessError - -import pytest -from pytest_mock import MockerFixture - -from betty.extension.npm import _NpmRequirement - - -class TestNpmRequirement: - async def test_check_met(self) -> None: - sut = _NpmRequirement.check() - assert sut.is_met() - - @pytest.mark.parametrize( - "e", - [ - CalledProcessError(1, ""), - FileNotFoundError(), - ], - ) - async def test_check_unmet(self, e: Exception, mocker: MockerFixture) -> None: - m_npm = mocker.patch("betty.extension.npm.npm") - m_npm.side_effect = e - sut = _NpmRequirement.check() - assert not sut.is_met() diff --git a/betty/tests/extension/trees/test___init__.py b/betty/tests/extension/trees/test___init__.py index 0611474c6..09714f40a 100644 --- a/betty/tests/extension/trees/test___init__.py +++ b/betty/tests/extension/trees/test___init__.py @@ -13,12 +13,18 @@ async def test_generate(self) -> None: app.project.configuration.extensions.append(ExtensionConfiguration(Trees)) await generate(app) async with aiofiles.open( - app.project.configuration.www_directory_path / "trees.js", encoding="utf-8" + app.project.configuration.www_directory_path + / "js" + / "betty.extension.Trees.js", + encoding="utf-8", ) as f: betty_js = await f.read() - assert "trees.js" in betty_js + assert Trees.name() in betty_js async with aiofiles.open( - app.project.configuration.www_directory_path / "trees.css", encoding="utf-8" + app.project.configuration.www_directory_path + / "css" + / "betty.extension.Trees.css", + encoding="utf-8", ) as f: betty_css = await f.read() - assert ".tree" in betty_css + assert Trees.name() in betty_css diff --git a/betty/tests/extension/webpack/__init__.py b/betty/tests/extension/webpack/__init__.py new file mode 100644 index 000000000..ac7c5ac94 --- /dev/null +++ b/betty/tests/extension/webpack/__init__.py @@ -0,0 +1 @@ +"""Test the :py:mod:`betty.extension.webpack` module.""" diff --git a/betty/tests/extension/webpack/test___init__.py b/betty/tests/extension/webpack/test___init__.py new file mode 100644 index 000000000..a32e3cd23 --- /dev/null +++ b/betty/tests/extension/webpack/test___init__.py @@ -0,0 +1,147 @@ +from pathlib import Path + +import aiofiles +import pytest +from aiofiles.os import makedirs +from pytest_mock import MockerFixture + +from betty import fs +from betty._npm import NpmUnavailable +from betty.app import App +from betty.extension.webpack import PrebuiltAssetsRequirement, Webpack +from betty.extension.webpack.build import webpack_build_id +from betty.generate import generate +from betty.job import Context +from betty.requirement import RequirementError + + +class TestPrebuiltAssetsRequirement: + @pytest.mark.parametrize( + "expected", + [ + True, + False, + ], + ) + async def test_is_met(self, expected: bool, tmp_path: Path) -> None: + prebuilt_assets_directory_path = tmp_path + if expected: + (prebuilt_assets_directory_path / "webpack").mkdir() + original_prebuilt_assets_directory_path = fs.PREBUILT_ASSETS_DIRECTORY_PATH + fs.PREBUILT_ASSETS_DIRECTORY_PATH = Path(prebuilt_assets_directory_path) + sut = PrebuiltAssetsRequirement() + try: + assert sut.is_met() is expected + finally: + fs.PREBUILT_ASSETS_DIRECTORY_PATH = original_prebuilt_assets_directory_path + + +class TestWebpack: + _SENTINEL = "s3nt1n3l" + + async def test_generate_with_npm( + self, mocker: MockerFixture, tmp_path: Path + ) -> None: + webpack_build_directory_path = tmp_path + m_build = mocker.patch("betty.extension.webpack.build.Builder.build") + m_build.return_value = webpack_build_directory_path + + async with aiofiles.open( + webpack_build_directory_path / self._SENTINEL, "w" + ) as f: + await f.write(self._SENTINEL) + + async with App.new_temporary() as app: + app.project.configuration.extensions.enable(Webpack) + await generate(app) + + async with aiofiles.open( + app.project.configuration.www_directory_path / self._SENTINEL + ) as f: + assert await f.read() == self._SENTINEL + + async def test_generate_without_npm_with_prebuild( + self, mocker: MockerFixture, tmp_path: Path + ) -> None: + m_build = mocker.patch("betty.extension.webpack.build.Builder.build") + m_build.side_effect = NpmUnavailable() + + webpack_build_directory_path = ( + tmp_path / "webpack" / f"build-{webpack_build_id(())}" + ) + await makedirs(webpack_build_directory_path) + async with aiofiles.open( + webpack_build_directory_path / self._SENTINEL, "w" + ) as f: + await f.write(self._SENTINEL) + + original_prebuilt_assets_directory_path = fs.PREBUILT_ASSETS_DIRECTORY_PATH + fs.PREBUILT_ASSETS_DIRECTORY_PATH = tmp_path + try: + async with App.new_temporary() as app: + app.project.configuration.extensions.enable(Webpack) + await generate(app) + finally: + fs.PREBUILT_ASSETS_DIRECTORY_PATH = original_prebuilt_assets_directory_path + + async with aiofiles.open( + app.project.configuration.www_directory_path / self._SENTINEL + ) as f: + assert await f.read() == self._SENTINEL + + async def test_generate_without_npm_without_prebuild( + self, mocker: MockerFixture, tmp_path: Path + ) -> None: + prebuilt_assets_directory_path = tmp_path + + m_build = mocker.patch("betty.extension.webpack.build.Builder.build") + m_build.side_effect = NpmUnavailable() + + original_prebuilt_assets_directory_path = fs.PREBUILT_ASSETS_DIRECTORY_PATH + fs.PREBUILT_ASSETS_DIRECTORY_PATH = ( + Path(prebuilt_assets_directory_path) / "does-not-exist" + ) + try: + async with App.new_temporary() as app: + app.project.configuration.extensions.enable(Webpack) + with pytest.raises(ExceptionGroup) as exc_info: + await generate(app) + error = exc_info.value + assert isinstance(error, ExceptionGroup) + assert error.subgroup(RequirementError) is not None + finally: + fs.PREBUILT_ASSETS_DIRECTORY_PATH = original_prebuilt_assets_directory_path + + async def test_prebuild(self, mocker: MockerFixture, tmp_path: Path) -> None: + webpack_build_directory_path = ( + tmp_path / "webpack" / f"build-{webpack_build_id(())}" + ) + prebuilt_assets_directory_path = tmp_path / "prebuild" + + m_build = mocker.patch("betty.extension.webpack.build.Builder.build") + m_build.return_value = webpack_build_directory_path + + await makedirs(webpack_build_directory_path) + async with aiofiles.open( + webpack_build_directory_path / self._SENTINEL, "w" + ) as f: + await f.write(self._SENTINEL) + + original_prebuilt_assets_directory_path = fs.PREBUILT_ASSETS_DIRECTORY_PATH + fs.PREBUILT_ASSETS_DIRECTORY_PATH = prebuilt_assets_directory_path + try: + job_context = Context() + async with App.new_temporary() as app: + app.project.configuration.extensions.enable(Webpack) + webpack = app.extensions[Webpack] + await webpack.prebuild(job_context) + finally: + fs.PREBUILT_ASSETS_DIRECTORY_PATH = original_prebuilt_assets_directory_path + + async with aiofiles.open( + prebuilt_assets_directory_path + / "webpack" + / f"build-{webpack_build_id(())}" + / self._SENTINEL + ) as f: + assert await f.read() == self._SENTINEL diff --git a/betty/tests/extension/webpack/test_build.py b/betty/tests/extension/webpack/test_build.py new file mode 100644 index 000000000..714ef9600 --- /dev/null +++ b/betty/tests/extension/webpack/test_build.py @@ -0,0 +1,104 @@ +from asyncio import to_thread +from collections.abc import Sequence +from pathlib import Path +from shutil import rmtree + +import pytest +from pytest_mock import MockerFixture + +from betty._npm import NpmUnavailable +from betty.app import App +from betty.app.extension import Extension +from betty.extension.webpack import WebpackEntrypointProvider +from betty.extension.webpack.build import Builder +from betty.job import Context +from betty.locale import DEFAULT_LOCALIZER + + +class DummyEntrypointProviderExtension(WebpackEntrypointProvider, Extension): + @classmethod + def webpack_entrypoint_directory_path(cls) -> Path: + return Path(__file__).parent / "test_build_webpack_entrypoint" + + def webpack_entrypoint_cache_keys(self) -> Sequence[str]: + return () + + +class TestBuilder: + @pytest.mark.parametrize( + "with_entrypoint_provider, debug, npm_install_cache_available, webpack_build_cache_available", + [ + (True, True, True, True), + (False, True, True, True), + (True, True, True, False), + (False, True, True, False), + (True, True, False, False), + (False, True, False, False), + (True, False, True, True), + (False, False, True, True), + (True, False, False, False), + (False, False, False, False), + (True, False, False, False), + (False, False, False, False), + ], + ) + async def test_build( + self, + with_entrypoint_provider: bool, + debug: bool, + npm_install_cache_available: bool, + tmp_path: Path, + webpack_build_cache_available: bool, + ) -> None: + async with App.new_temporary() as app: + if with_entrypoint_provider: + app.project.configuration.extensions.enable( + DummyEntrypointProviderExtension + ) + job_context = Context() + sut = Builder( + tmp_path, + ( + [app.extensions[DummyEntrypointProviderExtension]] + if with_entrypoint_provider + else [] + ), + False, + app.renderer, + job_context=job_context, + localizer=DEFAULT_LOCALIZER, + ) + if npm_install_cache_available: + webpack_build_directory_path = await sut.build() + if not webpack_build_cache_available: + await to_thread(rmtree, webpack_build_directory_path) + webpack_build_directory_path = await sut.build() + assert (webpack_build_directory_path / "css" / "vendor.css").exists() + assert ( + webpack_build_directory_path / "js" / "webpack-entry-loader.js" + ).exists() + if with_entrypoint_provider: + assert ( + webpack_build_directory_path + / "js" + / f"{DummyEntrypointProviderExtension.name()}.js" + ).exists() + + async def test_build_with_npm_unavailable( + self, mocker: MockerFixture, tmp_path: Path + ) -> None: + m_npm = mocker.patch("betty._npm.npm") + m_npm.side_effect = NpmUnavailable() + + job_context = Context() + m_renderer = mocker.AsyncMock() + sut = Builder( + tmp_path, + [], + False, + m_renderer, + job_context=job_context, + localizer=DEFAULT_LOCALIZER, + ) + with pytest.raises(NpmUnavailable): + await sut.build() diff --git a/betty/tests/extension/webpack/test_build_webpack_entrypoint/main.ts b/betty/tests/extension/webpack/test_build_webpack_entrypoint/main.ts new file mode 100644 index 000000000..e69de29bb diff --git a/betty/tests/extension/webpack/test_build_webpack_entrypoint/package.json b/betty/tests/extension/webpack/test_build_webpack_entrypoint/package.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/betty/tests/extension/webpack/test_build_webpack_entrypoint/package.json @@ -0,0 +1 @@ +{} diff --git a/betty/tests/test_npm.py b/betty/tests/test_npm.py new file mode 100644 index 000000000..5aa17111b --- /dev/null +++ b/betty/tests/test_npm.py @@ -0,0 +1,15 @@ +from pytest_mock import MockerFixture + +from betty._npm import NpmRequirement, NpmUnavailable + + +class TestNpmRequirement: + async def test_check_met(self) -> None: + sut = NpmRequirement() + assert sut.is_met() + + async def test_check_unmet(self, mocker: MockerFixture) -> None: + m_npm = mocker.patch("betty._npm.npm") + m_npm.side_effect = NpmUnavailable() + sut = NpmRequirement() + assert not sut.is_met() diff --git a/bin/build-setuptools b/bin/build-setuptools index dcdfb2425..7c1da018e 100755 --- a/bin/build-setuptools +++ b/bin/build-setuptools @@ -11,8 +11,6 @@ then fi echo "$1" > ./betty/assets/VERSION -rm -rf ./dist - # Install Python dependencies. pip install -e '.[setuptools]' @@ -20,7 +18,7 @@ pip install -e '.[setuptools]' npm install # Prepare the workspace directories. -rm -rf betty.egg-info build dist +./bin/clean-build # Build the package. python setup.py sdist diff --git a/bin/clean-build b/bin/clean-build index fceb89fc7..23848cfc1 100755 --- a/bin/clean-build +++ b/bin/clean-build @@ -4,5 +4,7 @@ set -Eeuo pipefail cd "$(dirname "$0")/.." +rm -rf ./build rm -rf ./dist -rm -rf ./betty/**/assets/betty.extension.npm._Npm/build +rm -rf ./betty.egg-info +rm -rf ./prebuild diff --git a/bin/fix b/bin/fix index c5e5de7d1..20091dceb 100755 --- a/bin/fix +++ b/bin/fix @@ -11,4 +11,4 @@ black . ./node_modules/.bin/stylelint --fix "./betty/**/*.css" # Fix JS code style violations. -./node_modules/.bin/eslint --fix -c ./eslint.config.js ./betty ./playwright \ No newline at end of file +./node_modules/.bin/eslint --fix -c ./eslint.config.js ./betty ./playwright diff --git a/eslint.config.js b/eslint.config.js index 8768c9367..2b1d11cbd 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -23,12 +23,14 @@ export default [ }, }, - // Betty extensions using the _Npm extension. + // The Webpack extension and other extensions using it. { files: [ - 'betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src/**', - 'betty/extension/maps/assets/betty.extension.npm._Npm/src/**', - 'betty/extension/trees/assets/betty.extension.npm._Npm/src/**', + 'betty/extension/cotton_candy/webpack/**', + 'betty/extension/http_api_doc/webpack/**', + 'betty/extension/maps/webpack/**', + 'betty/extension/trees/webpack/**', + 'betty/extension/webpack/webpack/**', ], languageOptions: { globals: { diff --git a/package.json b/package.json index 5ea27d99a..72dcfef83 100644 --- a/package.json +++ b/package.json @@ -6,10 +6,12 @@ "@playwright/test": "^1.43.1", "@stylistic/eslint-plugin": "^1.7.2", "@types/node": "*", - "betty.extension.CottonCandy": "file:./betty/extension/cotton_candy/assets/betty.extension.npm._Npm/src", - "betty.extension.HttpApiDoc": "file:./betty/extension/http_api_doc/assets/betty.extension.npm._Npm/src", - "betty.extension.Maps": "file:./betty/extension/maps/assets/betty.extension.npm._Npm/src", - "betty.extension.Trees": "file:./betty/extension/trees/assets/betty.extension.npm._Npm/src", + "@types/webpack": "^5.28.5", + "betty.extension.CottonCandy": "file:./betty/extension/cotton_candy/webpack", + "betty.extension.HttpApiDoc": "file:./betty/extension/http_api_doc/webpack", + "betty.extension.Maps": "file:./betty/extension/maps/webpack", + "betty.extension.Trees": "file:./betty/extension/trees/webpack", + "betty.extension.Webpack": "file:./betty/extension/webpack/webpack", "eslint": "^8.52.0", "eslint-config-standard": "^17.1.0", "eslint-plugin-compat": "^4.2.0", diff --git a/playwright.config.ts b/playwright.config.ts index 9f3e05445..2380fe76a 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -7,7 +7,7 @@ export default defineConfig({ fullyParallel: true, forbidOnly: !!process.env.CI, timeout: 60000, - retries: 9, + retries: process.env.CI ? 9 : 0, workers: parseInt(execSync('nproc').toString()), use: { trace: 'on-first-retry' diff --git a/pyproject.toml b/pyproject.toml index b20efea6c..dc4c4a946 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -153,8 +153,15 @@ exclude = [ [tool.setuptools.package-data] betty = [ 'py.typed', + + # Assets. 'assets/**', 'extension/*/assets/**', + + # Webpack. + '../.browserslistrc', + '../tsconfig.json', + 'extension/*/webpack/**', ] [tool.setuptools.exclude-package-data]