Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove multiprocessing #1226

Merged
merged 1 commit into from
Feb 24, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Remove multiprocessing
  • Loading branch information
bartfeenstra committed Feb 24, 2024
commit 402d1627bc35d5cca630c59829369bf2767d0bd9
2 changes: 0 additions & 2 deletions betty/_package/pyinstaller/main.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import sys
from multiprocessing import freeze_support

from betty.app import App
from betty.asyncio import sync
@@ -12,7 +11,6 @@ async def main() -> None:
"""
Launch Betty for PyInstaller builds.
"""
freeze_support()
async with App() as app:
qapp = BettyApplication([sys.argv[0]], app=app)
window = WelcomeWindow(app)
28 changes: 0 additions & 28 deletions betty/app/__init__.py
Original file line number Diff line number Diff line change
@@ -3,9 +3,7 @@
from __future__ import annotations

import operator
import os as stdos
import weakref
from collections.abc import Callable
from contextlib import suppress
from functools import reduce
from graphlib import CycleError, TopologicalSorter
@@ -157,21 +155,6 @@ def _unreduce(cls, dumped_app_configuration: VoidableDump, project: Project) ->
project,
)

def __reduce__(self) -> tuple[
Callable[[VoidableDump, Project], Self],
tuple[
VoidableDump,
Project,
],
]:
return (
App._unreduce,
(
self._configuration.dump(),
self._project,
),
)

async def __aenter__(self) -> Self:
await self.start()
return self
@@ -341,17 +324,6 @@ def renderer(self) -> Renderer:
def renderer(self) -> None:
self._renderer = None

@property
def concurrency(self) -> int:
with suppress(KeyError):
return int(stdos.environ['BETTY_CONCURRENCY'])
# Assume that any machine that runs Betty has at least two CPU cores.
return stdos.cpu_count() or 2

@property
def async_concurrency(self) -> int:
return self.concurrency ** 2

@property
def http_client(self) -> aiohttp.ClientSession:
if not self._http_client:
7 changes: 2 additions & 5 deletions betty/assets/betty.pot
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: Betty VERSION\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2024-02-17 18:26+0000\n"
"POT-Creation-Date: 2024-02-24 13:08+0000\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@@ -410,10 +410,7 @@ msgstr ""
msgid "Generate site"
msgstr ""

msgid "Generated pages for {count} {entity_type} in {locale}."
msgstr ""

msgid "Generated the listing page for {entity_type} in {locale}."
msgid "Generated {completed_job_count} out of {total_job_count} items ({completed_job_percentage}%)."
msgstr ""

msgid "Generating JSON Schema..."
11 changes: 5 additions & 6 deletions betty/assets/locale/de-DE/betty.po
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ msgid ""
msgstr ""
"Project-Id-Version: Betty VERSION\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2024-02-17 18:26+0000\n"
"POT-Creation-Date: 2024-02-24 13:08+0000\n"
"PO-Revision-Date: 2024-02-08 13:24+0000\n"
"Last-Translator: Bart Feenstra <bart@bartfeenstra.com>\n"
"Language: de\n"
@@ -574,11 +574,10 @@ msgstr "Seiten zur Auflistung von Entitäten generieren"
msgid "Generate site"
msgstr "Seite erzeugen"

msgid "Generated pages for {count} {entity_type} in {locale}."
msgstr "Generierte Seiten für {count} {entity_type} in {locale}."

msgid "Generated the listing page for {entity_type} in {locale}."
msgstr "Erzeugt die Auflistungsseite für {entity_type} in {locale}."
msgid ""
"Generated {completed_job_count} out of {total_job_count} items "
"({completed_job_percentage}%)."
msgstr ""

msgid "Generating JSON Schema..."
msgstr "Generiere JSON Schema..."
9 changes: 4 additions & 5 deletions betty/assets/locale/fr-FR/betty.po
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PROJECT VERSION\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2024-02-17 18:26+0000\n"
"POT-Creation-Date: 2024-02-24 13:08+0000\n"
"PO-Revision-Date: 2024-02-08 13:24+0000\n"
"Last-Translator: Bart Feenstra <bart@bartfeenstra.com>\n"
"Language: fr\n"
@@ -489,10 +489,9 @@ msgstr ""
msgid "Generate site"
msgstr ""

msgid "Generated pages for {count} {entity_type} in {locale}."
msgstr ""

msgid "Generated the listing page for {entity_type} in {locale}."
msgid ""
"Generated {completed_job_count} out of {total_job_count} items "
"({completed_job_percentage}%)."
msgstr ""

msgid "Generating JSON Schema..."
13 changes: 6 additions & 7 deletions betty/assets/locale/nl-NL/betty.po
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ msgid ""
msgstr ""
"Project-Id-Version: PROJECT VERSION\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2024-02-17 18:26+0000\n"
"POT-Creation-Date: 2024-02-24 13:08+0000\n"
"PO-Revision-Date: 2024-02-11 15:31+0000\n"
"Last-Translator: Bart Feenstra <bart@bartfeenstra.com>\n"
"Language: nl\n"
@@ -567,13 +567,12 @@ msgstr "Genereer pagina's met entiteitsoverzichten"
msgid "Generate site"
msgstr "Genereer site"

msgid "Generated pages for {count} {entity_type} in {locale}."
msgstr "Pagina's voor {count} {entity_type} in het {locale} gegenereerd."

msgid "Generated the listing page for {entity_type} in {locale}."
msgid ""
"Generated {completed_job_count} out of {total_job_count} items "
"({completed_job_percentage}%)."
msgstr ""
"Pagina met het overzicht van alle {entity_type} in het {locale} "
"gegenereerd."
"{completed_job_count} van de {total_job_count} items gegenereerd"
"({completed_job_percentage}%)."

msgid "Generating JSON Schema..."
msgstr "JSON Schema aan het genereren..."
9 changes: 4 additions & 5 deletions betty/assets/locale/uk/betty.po
Original file line number Diff line number Diff line change
@@ -7,7 +7,7 @@ msgid ""
msgstr ""
"Project-Id-Version: Betty VERSION\n"
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
"POT-Creation-Date: 2024-02-17 18:26+0000\n"
"POT-Creation-Date: 2024-02-24 13:08+0000\n"
"PO-Revision-Date: 2024-02-08 13:08+0000\n"
"Last-Translator: Rainer Thieringer <rainerthi@gmail.com>\n"
"Language: uk\n"
@@ -490,10 +490,9 @@ msgstr ""
msgid "Generate site"
msgstr ""

msgid "Generated pages for {count} {entity_type} in {locale}."
msgstr ""

msgid "Generated the listing page for {entity_type} in {locale}."
msgid ""
"Generated {completed_job_count} out of {total_job_count} items "
"({completed_job_percentage}%)."
msgstr ""

msgid "Generating JSON Schema..."
2 changes: 1 addition & 1 deletion betty/concurrent.py
Original file line number Diff line number Diff line change
@@ -10,7 +10,7 @@

class RateLimiter:
"""
Rate-limit tasks.
Rate-limit operations.

This class implements the `Token Bucket algorithm <https://en.wikipedia.org/wiki/Token_bucket>`_.
"""
6 changes: 3 additions & 3 deletions betty/extension/cotton_candy/__init__.py
Original file line number Diff line number Diff line change
@@ -24,7 +24,7 @@
from betty.functools import walk
from betty.generate import Generator, GenerationContext
from betty.gui import GuiBuilder
from betty.jinja2 import Jinja2Provider, context_app, context_localizer, context_task_context
from betty.jinja2 import Jinja2Provider, context_app, context_localizer, context_job_context
from betty.locale import Date, Str, Datey
from betty.model import Entity, UserFacingEntity, GeneratedEntityId
from betty.model.ancestry import Event, Person, Presence, is_public, Subject
@@ -228,7 +228,7 @@ async def _copy_npm_build(self, source_directory_path: Path, destination_directo
await asyncio.to_thread(copy2, source_directory_path / 'cotton_candy.css', destination_directory_path / 'cotton_candy.css')
await asyncio.to_thread(copy2, source_directory_path / 'cotton_candy.js', destination_directory_path / 'cotton_candy.js')

async def generate(self, task_context: GenerationContext) -> None:
async def generate(self, job_context: GenerationContext) -> None:
assets_directory_path = await self.app.extensions[_Npm].ensure_assets(self)
await makedirs(self.app.project.configuration.www_directory_path, exist_ok=True)
await self._copy_npm_build(assets_directory_path, self.app.project.configuration.www_directory_path)
@@ -238,7 +238,7 @@ async def generate(self, task_context: GenerationContext) -> None:
async def _global_search_index(context: Context) -> AsyncIterable[dict[str, str]]:
return Index(
context_app(context),
context_task_context(context),
context_job_context(context),
context_localizer(context),
).build()

8 changes: 4 additions & 4 deletions betty/extension/cotton_candy/search.py
Original file line number Diff line number Diff line change
@@ -11,18 +11,18 @@
from betty.model import get_entity_type_name, Entity
from betty.model.ancestry import Person, Place, File
from betty.string import camel_case_to_snake_case
from betty.task import Context
from betty.job import Context


class Index:
def __init__(
self,
app: App,
task_context: Context | None,
job_context: Context | None,
localizer: Localizer,
):
self._app = app
self._task_context = task_context
self._job_context = job_context
self._localizer = localizer

async def build(self) -> AsyncIterable[dict[str, str]]:
@@ -57,7 +57,7 @@ async def _render_entity(self, entity: Entity) -> str:
f'search/result-{camel_case_to_snake_case(entity_type_name)}.html.j2',
'search/result.html.j2',
]).render_async({
'task_context': self._task_context,
'job_context': self._job_context,
'localizer': self._localizer,
'entity': entity,
})
2 changes: 1 addition & 1 deletion betty/extension/http_api_doc/__init__.py
Original file line number Diff line number Diff line change
@@ -29,7 +29,7 @@ async def npm_build(self, working_directory_path: Path, assets_directory_path: P
def npm_cache_scope(cls) -> CacheScope:
return CacheScope.BETTY

async def generate(self, task_context: GenerationContext) -> None:
async def generate(self, job_context: GenerationContext) -> None:
assets_directory_path = await self.app.extensions[_Npm].ensure_assets(self)
await makedirs(self.app.project.configuration.www_directory_path, exist_ok=True)
await asyncio.to_thread(copy2, assets_directory_path / 'http-api-doc.js', self.app.project.configuration.www_directory_path / 'http-api-doc.js')
2 changes: 1 addition & 1 deletion betty/extension/maps/__init__.py
Original file line number Diff line number Diff line change
@@ -39,7 +39,7 @@ async def _copy_npm_build(self, source_directory_path: Path, destination_directo
def npm_cache_scope(cls) -> CacheScope:
return CacheScope.BETTY

async def generate(self, task_context: GenerationContext) -> None:
async def generate(self, job_context: GenerationContext) -> None:
assets_directory_path = await self.app.extensions[_Npm].ensure_assets(self)
await makedirs(self.app.project.configuration.www_directory_path, exist_ok=True)
await self._copy_npm_build(assets_directory_path, self.app.project.configuration.www_directory_path)
2 changes: 1 addition & 1 deletion betty/extension/nginx/__init__.py
Original file line number Diff line number Diff line change
@@ -103,7 +103,7 @@ def servers(self) -> Sequence[Server]:
return [DockerizedNginxServer(self._app)]
return []

async def generate(self, task_context: GenerationContext) -> None:
async def generate(self, job_context: GenerationContext) -> None:
await generate_configuration_file(self._app)
await generate_dockerfile_file(self._app)

23 changes: 12 additions & 11 deletions betty/extension/nginx/serve.py
Original file line number Diff line number Diff line change
@@ -5,29 +5,34 @@
from pathlib import Path
from typing import Any

import dill
import docker
from aiofiles.tempfile import TemporaryDirectory, AiofilesContextManagerTempDir
from docker.errors import DockerException

from betty.app import App
from betty.extension.nginx.artifact import generate_dockerfile_file, generate_configuration_file
from betty.extension.nginx.docker import Container
from betty.project import Project
from betty.serve import NoPublicUrlBecauseServerNotStartedError, AppServer


class DockerizedNginxServer(AppServer):
def __init__(self, app: App) -> None:
super().__init__(
# Create a new app so we can modify it later.
dill.loads(dill.dumps(app))
)
from betty.extension import Nginx

project = Project(ancestry=app.project.ancestry)
project.configuration.autowrite = False
project.configuration.configuration_file_path = app.project.configuration.configuration_file_path
project.configuration.update(app.project.configuration)
project.configuration.debug = True
app = App(app.configuration, project)
# Work around https://github.com/bartfeenstra/betty/issues/1056.
app.extensions[Nginx].configuration.https = False
super().__init__(app)
self._container: Container | None = None
self._output_directory: AiofilesContextManagerTempDir[None, Any, Any] | None = None

async def start(self) -> None:
from betty.extension import Nginx

await super().start()
logging.getLogger(__name__).info('Starting a Dockerized nginx web server...')
self._output_directory = TemporaryDirectory()
@@ -36,10 +41,6 @@ async def start(self) -> None:
docker_directory_path = Path(output_directory_name)
dockerfile_file_path = docker_directory_path / 'Dockerfile'

self._app.project.configuration.debug = True
# Work around https://github.com/bartfeenstra/betty/issues/1056.
self._app.extensions[Nginx].configuration.https = False

await generate_configuration_file(
self._app,
destination_file_path=nginx_configuration_file_path,
2 changes: 1 addition & 1 deletion betty/extension/trees/__init__.py
Original file line number Diff line number Diff line change
@@ -36,7 +36,7 @@ async def _copy_npm_build(self, source_directory_path: Path, destination_directo
def npm_cache_scope(cls) -> CacheScope:
return CacheScope.BETTY

async def generate(self, task_context: GenerationContext) -> None:
async def generate(self, job_context: GenerationContext) -> None:
assets_directory_path = await self.app.extensions[_Npm].ensure_assets(self)
await self._copy_npm_build(assets_directory_path, self.app.project.configuration.www_directory_path)

Loading
Loading