From d9d18072dc1afd9b79fb14d32593f74cc376e425 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Wed, 23 Nov 2022 16:08:39 +0100 Subject: [PATCH 01/16] :building_construction: [#1068] -- merge formio component registries into single registry --- src/openforms/formio/registry.py | 128 +++++++++++++++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 src/openforms/formio/registry.py diff --git a/src/openforms/formio/registry.py b/src/openforms/formio/registry.py new file mode 100644 index 0000000000..71ffa34c5a --- /dev/null +++ b/src/openforms/formio/registry.py @@ -0,0 +1,128 @@ +""" +Define a single registry for all Formio component types. + +Various aspects for formio components are registered in a single place, such as: + +* normalization +* formatting of values +* bringing in the render node information + +This allows us to treat all aspects of every component type together rather than +smeared out across the codebase in similar but different implementations, while making +the public API better defined and smaller. +""" +from typing import TYPE_CHECKING, Any, Protocol + +from openforms.plugins.plugin import AbstractBasePlugin +from openforms.plugins.registry import BaseRegistry +from openforms.typing import DataMapping + +from .datastructures import FormioConfigurationWrapper +from .typing import Component + +if TYPE_CHECKING: + from openforms.submissions.models import Submission + + +class FormatterProtocol(Protocol): + multiple_separator: str = "; " + """ + Separator to use for multi-value components. + + Defaults to semi-colon, as formatted numbers already use comma's which hurts + readability. + """ + as_html = False + """ + Format for HTML output or not. + + The default is to format for plain text output, but toggling this will emit + HTML where relevant. + """ + + # there is an interesting open question on what to do for empty values + # currently we're eating them in normalise_value_to_list() + empty_values = [None, ""] + + def __call__(self, component: Component, value: Any, as_html=False) -> str: + ... + + def format(self, component: Component, value: Any) -> str: + """ + Format a single value for the given component. + """ + ... + + +class BasePlugin(AbstractBasePlugin): + """ + Base class for Formio component plugins. + """ + + is_enabled: bool = True + + formatter: type[FormatterProtocol] + """ + Specify the callable to use for formatting. + + Formatter (class) implementation, used by + :meth:`openforms.formio.registry.ComponentRegistry.format`. + """ + + @staticmethod + def mutate(component: Component, data: DataMapping) -> None: # pragma: nocover + ... + + +class ComponentRegistry(BaseRegistry): + module = "formio_components" + + def format(self, info: Component, value: Any, as_html=False): + """ + Format a given value in the appropriate way for the specified component. + + This results in a human-readable string representation of the value submitted + for the given component type, as it makes the best sense for that component + type. + """ + formatter = ( + register[info["type"]] if info["type"] in register else register["default"] + ) + return formatter(info, value, as_html=as_html) + + def update_config( + self, component: Component, data: DataMapping | None = None + ) -> None: + """ + Mutate the component configuration in place. + + Mutating the config in place allows dynamic configurations (because of logic, + for example) to work. + """ + # if there is no plugin registered for the component, return the input + if (component_type := component["type"]) not in register: + return + + # invoke plugin if exists + plugin = self[component_type] + plugin.mutate(component, data) + + def handle_custom_types( + self, + configuration: FormioConfigurationWrapper, + submission: "Submission", + ): + """ + Process custom backend-only formio types. + + Formio types can be transformed in the context of a given + :class:`openforms.submissions.models.Submission` and ultimately manifest as + modified or standard Formio types, essentially performing some sort of "rewrite" + of the Formio configuration object. + """ + raise NotImplementedError() + + +# Sentinel to provide the default registry. You can easily instantiate another +# :class:`Registry` object to use as dependency injection in tests. +register = ComponentRegistry() From be6c34652247fdbd12af69cf2c743684ac5e3336 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Wed, 23 Nov 2022 16:57:27 +0100 Subject: [PATCH 02/16] :recycle: [#1068] -- Refactor formio formatters into component plugin The public API now uses the new/single registry for formio components, which also handles the formatting of values given to formio components. --- src/openforms/formio/apps.py | 12 ++ src/openforms/formio/components/__init__.py | 2 + src/openforms/formio/components/custom.py | 7 + src/openforms/formio/components/vanilla.py | 109 ++++++++++++ src/openforms/formio/formatters/base.py | 74 +++++++++ src/openforms/formio/formatters/custom.py | 10 +- src/openforms/formio/formatters/formio.py | 173 ++++++++++++++++++++ src/openforms/formio/formatters/service.py | 7 +- src/openforms/formio/registry.py | 44 ++--- src/openforms/formio/typing.py | 3 +- 10 files changed, 406 insertions(+), 35 deletions(-) create mode 100644 src/openforms/formio/apps.py create mode 100644 src/openforms/formio/components/__init__.py create mode 100644 src/openforms/formio/components/custom.py create mode 100644 src/openforms/formio/components/vanilla.py create mode 100644 src/openforms/formio/formatters/base.py create mode 100644 src/openforms/formio/formatters/formio.py diff --git a/src/openforms/formio/apps.py b/src/openforms/formio/apps.py new file mode 100644 index 0000000000..ba0d6fbf9a --- /dev/null +++ b/src/openforms/formio/apps.py @@ -0,0 +1,12 @@ +from django.apps import AppConfig +from django.utils.translation import gettext_lazy as _ + + +class FormioConfig(AppConfig): + name = "openforms.formio" + verbose_name = _("Formio integration") + + def ready(self): + from .components import register_all + + register_all() diff --git a/src/openforms/formio/components/__init__.py b/src/openforms/formio/components/__init__.py new file mode 100644 index 0000000000..e82780cdbe --- /dev/null +++ b/src/openforms/formio/components/__init__.py @@ -0,0 +1,2 @@ +def register_all(): + from . import custom, vanilla # noqa diff --git a/src/openforms/formio/components/custom.py b/src/openforms/formio/components/custom.py new file mode 100644 index 0000000000..ed95c8de2f --- /dev/null +++ b/src/openforms/formio/components/custom.py @@ -0,0 +1,7 @@ +from ..formatters.custom import MapFormatter +from ..registry import BasePlugin, register + + +@register("map") +class Map(BasePlugin): + formatter = MapFormatter diff --git a/src/openforms/formio/components/vanilla.py b/src/openforms/formio/components/vanilla.py new file mode 100644 index 0000000000..3253023f7b --- /dev/null +++ b/src/openforms/formio/components/vanilla.py @@ -0,0 +1,109 @@ +""" +Implement backend functionality for core Formio (built-in) component types. + +Custom component types (defined by us or third parties) need to be organized in the +adjacent custom.py module. +""" +from ..formatters.formio import ( + CheckboxFormatter, + CurrencyFormatter, + DateFormatter, + DefaultFormatter, + EmailFormatter, + FileFormatter, + NumberFormatter, + PasswordFormatter, + PhoneNumberFormatter, + RadioFormatter, + SelectBoxesFormatter, + SelectFormatter, + SignatureFormatter, + TextAreaFormatter, + TextFieldFormatter, + TimeFormatter, +) +from ..registry import BasePlugin, register + + +@register("default") +class Default(BasePlugin): + """ + Fallback for unregistered component types, implementing default behaviour. + """ + + formatter = DefaultFormatter + + +@register("textfield") +class TextField(BasePlugin): + formatter = TextFieldFormatter + + +@register("email") +class Email(BasePlugin): + formatter = EmailFormatter + + +@register("date") +class Date(BasePlugin): + formatter = DateFormatter + + +@register("time") +class Time(BasePlugin): + formatter = TimeFormatter + + +@register("phoneNumber") +class PhoneNumber(BasePlugin): + formatter = PhoneNumberFormatter + + +@register("file") +class File(BasePlugin): + formatter = FileFormatter + + +@register("textarea") +class TextArea(BasePlugin): + formatter = TextAreaFormatter + + +@register("number") +class Number(BasePlugin): + formatter = NumberFormatter + + +@register("password") +class Password(BasePlugin): + formatter = PasswordFormatter + + +@register("checkbox") +class Checkbox(BasePlugin): + formatter = CheckboxFormatter + + +@register("selectboxes") +class SelectBoxes(BasePlugin): + formatter = SelectBoxesFormatter + + +@register("select") +class Select(BasePlugin): + formatter = SelectFormatter + + +@register("currency") +class Currency(BasePlugin): + formatter = CurrencyFormatter + + +@register("radio") +class Radio(BasePlugin): + formatter = RadioFormatter + + +@register("signature") +class Signature(BasePlugin): + formatter = SignatureFormatter diff --git a/src/openforms/formio/formatters/base.py b/src/openforms/formio/formatters/base.py new file mode 100644 index 0000000000..91964b6292 --- /dev/null +++ b/src/openforms/formio/formatters/base.py @@ -0,0 +1,74 @@ +import logging +from dataclasses import dataclass +from typing import Any, Iterable, Sequence + +from django.utils.encoding import force_str +from django.utils.html import format_html_join + +from ..typing import Component + +logger = logging.getLogger(__name__) + + +@dataclass +class FormatterBase: + as_html: bool = False + """ + Format for HTML output or not. + + The default is to format for plain text output, but toggling this will emit + HTML where relevant. + """ + multiple_separator: str = "; " + """ + Separator to use for multi-value components. + + Defaults to semi-colon, as formatted numbers already use comma's which hurts + readability. + """ + + # there is an interesting open question on what to do for empty values + # currently we're eating them in normalise_value_to_list() + empty_values: Sequence[Any] = (None, "") + + def is_empty_value(self, component: Component, value: Any): + return value in self.empty_values + + def normalise_value_to_list(self, component: Component, value: Any): + multiple = component.get("multiple", False) + # this breaks if multiple is true and value not a list + if multiple: + if not isinstance(value, (tuple, list)): + # this happens if value is None + value = [value] + else: + value = [value] + # convert to list of useful values + return [v for v in value if not self.is_empty_value(component, v)] + + def join_formatted_values( + self, component: Component, formatted_values: Iterable[str] + ) -> str: + if self.as_html: + args_generator = ((formatted,) for formatted in formatted_values) + return format_html_join(self.multiple_separator, "{}", args_generator) + else: + return self.multiple_separator.join(formatted_values) + + def process_result(self, component: Component, formatted: str) -> str: + return formatted + + def __call__(self, component: Component, value: Any) -> str: + # note all this depends on value not being unexpected type or shape + values = self.normalise_value_to_list(component, value) + + formatted_values = ( + force_str(self.format(component, value)) for value in values + ) + # logically we'd want a .filter_formatted_values() step here + return self.process_result( + component, self.join_formatted_values(component, formatted_values) + ) + + def format(self, component: Component, value: Any) -> str: + raise NotImplementedError("%r must implement the 'format' method" % type(self)) diff --git a/src/openforms/formio/formatters/custom.py b/src/openforms/formio/formatters/custom.py index efd086dc11..7732156ec3 100644 --- a/src/openforms/formio/formatters/custom.py +++ b/src/openforms/formio/formatters/custom.py @@ -1 +1,9 @@ -# TODO implement: iban, bsn, postcode, licenseplate, npFamilyMembers, cosign, map +# TODO implement: iban, bsn, postcode, licenseplate, npFamilyMembers, cosign +from ..typing import Component +from .base import FormatterBase + + +class MapFormatter(FormatterBase): + def format(self, component: Component, value: list[float]) -> str: + # use a comma here since its a single data element + return ", ".join((str(x) for x in value)) diff --git a/src/openforms/formio/formatters/formio.py b/src/openforms/formio/formatters/formio.py new file mode 100644 index 0000000000..970795f873 --- /dev/null +++ b/src/openforms/formio/formatters/formio.py @@ -0,0 +1,173 @@ +import logging +from datetime import datetime +from typing import Any + +from django.template.defaultfilters import date as fmt_date, time as fmt_time, yesno +from django.utils.dateparse import parse_date, parse_time +from django.utils.formats import number_format +from django.utils.html import format_html +from django.utils.translation import gettext_lazy as _ + +from glom import glom + +from ..typing import Component, OptionDict +from .base import FormatterBase + +logger = logging.getLogger(__name__) + + +def get_value_label(possible_values: list[OptionDict], value: int | str) -> str: + # From #1466 it's clear that Formio does not force the values to be strings, e.g. + # if you use numeric values for the options. They are stored as string in the form + # configuration, but the submitted value is a number. + # See https://github.com/formio/formio.js/blob/4.12.x/src/components/radio/Radio.js#L208 + # (unmodified in 4.13) but also normalization in the select component + # https://github.com/formio/formio.js/blob/4.12.x/src/components/select/Select.js#L1227 + _original = value + # cast to string to compare against the values + if not isinstance(value, str): + value = str(value) + logger.info( + "Casted original value %r to string value %s for option comparison", + _original, + value, + ) + + for possible_value in possible_values: + if possible_value["value"] == value: + return possible_value["label"] + + return value + + +class DefaultFormatter(FormatterBase): + def format(self, component: Component, value: Any) -> str: + return str(value) + + +class TextFieldFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + return str(value) + + +class EmailFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + return str(value) + + +class DateFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + return fmt_date(parse_date(value)) + + +class TimeFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + return fmt_time(parse_time(value)) + + +class PhoneNumberFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + # TODO custom formatting? + return str(value) + + +class FileFormatter(FormatterBase): + def normalise_value_to_list(self, component: Component, value: Any): + if value is None: + return [] + else: + # file component is always a list + return value + + def process_result(self, component: Component, formatted: str) -> str: + # prefix joined filenames to match legacy + if formatted: + return _("attachment: %s") % formatted + return formatted + + def format(self, component: Component, value: dict) -> str: + # this is only valid for display to the user (because filename component option, dedupe etc) + return value["originalName"] + + +class TextAreaFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + # TODO custom formatting? + return str(value) + + +class NumberFormatter(FormatterBase): + def format(self, component: Component, value: int | float) -> str: + # localized and forced to decimalLimit + return number_format(value, decimal_pos=component.get("decimalLimit")) + + +class PasswordFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + # TODO legacy just printed as-is, but we might want to use unicode-dots or stars + # return "\u25CF" * len(value) + return str(value) + + +class CheckboxFormatter(FormatterBase): + def format(self, component: Component, value: bool) -> str: + return str(yesno(value)) + + +class SelectBoxesFormatter(FormatterBase): + def format(self, component: Component, value: dict[str, bool]) -> str: + selected_labels = [ + entry["label"] for entry in component["values"] if value.get(entry["value"]) + ] + return self.multiple_separator.join(selected_labels) + + +class SelectFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + # grab appointment specific data + if glom(component, "appointments.showDates", default=False): + return fmt_date(parse_date(value)) + elif glom(component, "appointments.showTimes", default=False): + # strip the seconds from a full ISO datetime + return fmt_time(datetime.fromisoformat(value)) + elif glom(component, "appointments.showLocations", default=False) or glom( + component, "appointments.showProducts", default=False + ): + if isinstance(value, dict): + return value["name"] + else: + # shouldn't happen + return str(value) + else: + # regular value select + return get_value_label(component["data"]["values"], value) + + +class CurrencyFormatter(FormatterBase): + def format(self, component: Component, value: float) -> str: + # localized and forced to decimalLimit + # note we mirror formio and default to 2 decimals + return number_format(value, decimal_pos=component.get("decimalLimit", 2)) + + +class RadioFormatter(FormatterBase): + def format(self, component: Component, value: str | int) -> str: + return get_value_label(component["values"], value) + + +class SignatureFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + text = _("signature added") + if not self.as_html: + return text + + assert value.startswith( + "data:image/" + ), "Expected 'data:' URI with image mime type" + + # max-width is required for e-mail styling where it may overflow a table cell + return format_html( + """{alt}""", + src=value, + alt=text, + ) diff --git a/src/openforms/formio/formatters/service.py b/src/openforms/formio/formatters/service.py index bf3860c155..8251a112cc 100644 --- a/src/openforms/formio/formatters/service.py +++ b/src/openforms/formio/formatters/service.py @@ -1,11 +1,12 @@ from typing import Any +from ..registry import register from ..typing import Component from .printable import filter_printable -from .registry import register __all__ = ["format_value", "filter_printable"] -def format_value(info: Component, value: Any, as_html=False): - return register.format(info, value, as_html=as_html) +# TODO: move to parent level service module and deprecate this one. +def format_value(component: Component, value: Any, as_html=False): + return register.format(component, value, as_html=as_html) diff --git a/src/openforms/formio/registry.py b/src/openforms/formio/registry.py index 71ffa34c5a..397ce4442a 100644 --- a/src/openforms/formio/registry.py +++ b/src/openforms/formio/registry.py @@ -25,32 +25,10 @@ class FormatterProtocol(Protocol): - multiple_separator: str = "; " - """ - Separator to use for multi-value components. - - Defaults to semi-colon, as formatted numbers already use comma's which hurts - readability. - """ - as_html = False - """ - Format for HTML output or not. - - The default is to format for plain text output, but toggling this will emit - HTML where relevant. - """ - - # there is an interesting open question on what to do for empty values - # currently we're eating them in normalise_value_to_list() - empty_values = [None, ""] - - def __call__(self, component: Component, value: Any, as_html=False) -> str: + def __init__(self, as_html: bool): ... - def format(self, component: Component, value: Any) -> str: - """ - Format a single value for the given component. - """ + def __call__(self, component: Component, value: Any) -> str: ... @@ -77,7 +55,13 @@ def mutate(component: Component, data: DataMapping) -> None: # pragma: nocover class ComponentRegistry(BaseRegistry): module = "formio_components" - def format(self, info: Component, value: Any, as_html=False): + def normalize(self, component: Component, value: Any) -> Any: + """ + Given a value from any source, normalize it according to the component rules. + """ + raise NotImplementedError() + + def format(self, component: Component, value: Any, as_html=False): """ Format a given value in the appropriate way for the specified component. @@ -85,10 +69,12 @@ def format(self, info: Component, value: Any, as_html=False): for the given component type, as it makes the best sense for that component type. """ - formatter = ( - register[info["type"]] if info["type"] in register else register["default"] - ) - return formatter(info, value, as_html=as_html) + if (component_type := component["type"]) not in self: + component_type = "default" + + component_plugin = self[component_type] + formatter = component_plugin.formatter(as_html=as_html) + return formatter(component, value) def update_config( self, component: Component, data: DataMapping | None = None diff --git a/src/openforms/formio/typing.py b/src/openforms/formio/typing.py index 0e3b5205c3..b7a3ca0e15 100644 --- a/src/openforms/formio/typing.py +++ b/src/openforms/formio/typing.py @@ -39,8 +39,7 @@ class Component(TypedDict): We deliberately document keys here that may be absent, because: * we don't run mypy (yet) and type hints are used as just hints/documentation - * the mechanism to define this correctly is not fleshed out, see also - https://discuss.python.org/t/pep-655-required-and-notrequired-for-typeddict/13817/3 + * NotRequired is only available in typing_extensions and Python 3.11+ """ type: str From 53d7d118ed11e75f58e818f027156fa0abf2c011 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Wed, 23 Nov 2022 17:00:49 +0100 Subject: [PATCH 03/16] :fire: [#1068] -- remove standalone formio.formatters app code This is now used by the component registry rather than implementing/ providing a formatter registry itself. --- src/openforms/conf/base.py | 1 - src/openforms/formio/formatters/apps.py | 12 - src/openforms/formio/formatters/default.py | 270 -------------------- src/openforms/formio/formatters/registry.py | 22 -- 4 files changed, 305 deletions(-) delete mode 100644 src/openforms/formio/formatters/apps.py delete mode 100644 src/openforms/formio/formatters/default.py delete mode 100644 src/openforms/formio/formatters/registry.py diff --git a/src/openforms/conf/base.py b/src/openforms/conf/base.py index 88e8f02e7e..115ef07aa7 100644 --- a/src/openforms/conf/base.py +++ b/src/openforms/conf/base.py @@ -194,7 +194,6 @@ "openforms.emails", "openforms.formio", "openforms.formio.dynamic_config", - "openforms.formio.formatters.apps.FormIOFormattersApp", "openforms.formio.rendering", "openforms.forms", "openforms.multidomain", diff --git a/src/openforms/formio/formatters/apps.py b/src/openforms/formio/formatters/apps.py deleted file mode 100644 index fddbecc315..0000000000 --- a/src/openforms/formio/formatters/apps.py +++ /dev/null @@ -1,12 +0,0 @@ -from django.apps import AppConfig -from django.utils.translation import gettext_lazy as _ - - -class FormIOFormattersApp(AppConfig): - name = "openforms.formio.formatters" - label = "formio_formatters" - verbose_name = _("FormIO formatters") - - def ready(self): - # register the plugin - from . import default # noqa diff --git a/src/openforms/formio/formatters/default.py b/src/openforms/formio/formatters/default.py deleted file mode 100644 index 54a68a92fe..0000000000 --- a/src/openforms/formio/formatters/default.py +++ /dev/null @@ -1,270 +0,0 @@ -import logging -from datetime import datetime -from typing import Any, Dict, Iterable, List, Union - -from django.template.defaultfilters import date as fmt_date, time as fmt_time, yesno -from django.utils.dateparse import parse_date, parse_time -from django.utils.encoding import force_str -from django.utils.formats import number_format -from django.utils.html import format_html, format_html_join -from django.utils.translation import gettext_lazy as _ - -from glom import glom - -from openforms.plugins.plugin import AbstractBasePlugin - -from ..typing import Component, OptionDict -from .registry import register - -logger = logging.getLogger(__name__) - - -def get_value_label(possible_values: List[OptionDict], value: Union[int, str]) -> str: - # From #1466 it's clear that Formio does not force the values to be strings, e.g. - # if you use numeric values for the options. They are stored as string in the form - # configuration, but the submitted value is a number. - # See https://github.com/formio/formio.js/blob/4.12.x/src/components/radio/Radio.js#L208 - # (unmodified in 4.13) but also normalization in the select component - # https://github.com/formio/formio.js/blob/4.12.x/src/components/select/Select.js#L1227 - _original = value - # cast to string to compare against the values - if not isinstance(value, str): - value = str(value) - logger.info( - "Casted original value %r to string value %s for option comparison", - _original, - value, - ) - - for possible_value in possible_values: - if possible_value["value"] == value: - return possible_value["label"] - - return value - - -def join_mapped(value: Any, formatter: callable = str, separator: str = ", ") -> str: - # map multiple value into a joined string - # note: filter before passing to this - # TODO if this stays this simple after the refactoring we can inline and remove it - return separator.join(map(formatter, value)) - - -class FormioFormatter(AbstractBasePlugin): - multiple_separator = "; " - """ - Separator to use for multi-value components. - - Defaults to semi-colon, as formatted numbers already use comma's which hurts - readability. - """ - as_html = False - """ - Format for HTML output or not. - - The default is to format for plain text output, but toggling this will emit - HTML where relevant. - """ - - # there is an interesting open question on what to do for empty values - # currently we're eating them in normalise_value_to_list() - empty_values = [None, ""] - - def is_empty_value(self, component: Component, value: Any): - return value in self.empty_values - - def normalise_value_to_list(self, component: Component, value: Any): - multiple = component.get("multiple", False) - # this breaks if multiple is true and value not a list - if multiple: - if not isinstance(value, (tuple, list)): - # this happens if value is None - value = [value] - else: - value = [value] - # convert to list of useful values - return [v for v in value if not self.is_empty_value(component, v)] - - def join_formatted_values( - self, component: Component, formatted_values: Iterable[str] - ) -> str: - if self.as_html: - args_generator = ((formatted,) for formatted in formatted_values) - return format_html_join(self.multiple_separator, "{}", args_generator) - else: - return self.multiple_separator.join(formatted_values) - - def process_result(self, component: Component, formatted: str) -> str: - return formatted - - def __call__(self, component: Component, value: Any, as_html=False) -> str: - self.as_html = as_html - # note all this depends on value not being unexpected type or shape - values = self.normalise_value_to_list(component, value) - - formatted_values = ( - force_str(self.format(component, value)) for value in values - ) - # logically we'd want a .filter_formatted_values() step here - return self.process_result( - component, self.join_formatted_values(component, formatted_values) - ) - - def format(self, component: Component, value: Any) -> str: - raise NotImplementedError("%r must implement the 'format' method" % type(self)) - - -@register("default") -class DefaultFormatter(FormioFormatter): - def format(self, component: Component, value: Any) -> str: - return str(value) - - -@register("textfield") -class TextFieldFormatter(FormioFormatter): - def format(self, component: Component, value: str) -> str: - return str(value) - - -@register("email") -class EmailFormatter(FormioFormatter): - def format(self, component: Component, value: str) -> str: - return str(value) - - -@register("date") -class DateFormatter(FormioFormatter): - def format(self, component: Component, value: str) -> str: - return fmt_date(parse_date(value)) - - -@register("time") -class TimeFormatter(FormioFormatter): - def format(self, component: Component, value: str) -> str: - return fmt_time(parse_time(value)) - - -@register("phoneNumber") -class PhoneNumberFormatter(FormioFormatter): - def format(self, component: Component, value: str) -> str: - # TODO custom formatting? - return str(value) - - -@register("file") -class FileFormatter(FormioFormatter): - def normalise_value_to_list(self, component: Component, value: Any): - if value is None: - return [] - else: - # file component is always a list - return value - - def process_result(self, component: Component, formatted: str) -> str: - # prefix joined filenames to match legacy - if formatted: - return _("attachment: %s") % formatted - return formatted - - def format(self, component: Component, value: List) -> str: - # this is only valid for display to the user (because filename component option, dedupe etc) - return value["originalName"] - - -@register("textarea") -class TextAreaFormatter(FormioFormatter): - def format(self, component: Component, value: str) -> str: - # TODO custom formatting? - return str(value) - - -@register("number") -class NumberFormatter(FormioFormatter): - def format(self, component: Component, value: Union[int, float]) -> str: - # localized and forced to decimalLimit - return number_format(value, decimal_pos=component.get("decimalLimit")) - - -@register("password") -class PasswordFormatter(FormioFormatter): - def format(self, component: Component, value: str) -> str: - # TODO legacy just printed as-is, but we might want to use unicode-dots or stars - # return "\u25CF" * len(value) - return str(value) - - -@register("checkbox") -class CheckboxFormatter(FormioFormatter): - def format(self, component: Component, value: bool) -> str: - return yesno(value) - - -@register("selectboxes") -class SelectBoxesFormatter(FormioFormatter): - def format(self, component: Component, value: Dict[str, bool]) -> str: - selected_labels = [ - entry["label"] for entry in component["values"] if value.get(entry["value"]) - ] - return self.multiple_separator.join(selected_labels) - - -@register("select") -class SelectFormatter(FormioFormatter): - def format(self, component: Component, value: str) -> str: - # grab appointment specific data - if glom(component, "appointments.showDates", default=False): - return fmt_date(parse_date(value)) - elif glom(component, "appointments.showTimes", default=False): - # strip the seconds from a full ISO datetime - return fmt_time(datetime.fromisoformat(value)) - elif glom(component, "appointments.showLocations", default=False) or glom( - component, "appointments.showProducts", default=False - ): - if isinstance(value, dict): - return value["name"] - else: - # shouldn't happen - return str(value) - else: - # regular value select - return get_value_label(component["data"]["values"], value) - - -@register("currency") -class CurrencyFormatter(FormioFormatter): - def format(self, component: Component, value: float) -> str: - # localized and forced to decimalLimit - # note we mirror formio and default to 2 decimals - return number_format(value, decimal_pos=component.get("decimalLimit", 2)) - - -@register("radio") -class RadioFormatter(FormioFormatter): - def format(self, component: Component, value: Union[str, int]) -> str: - return get_value_label(component["values"], value) - - -@register("signature") -class SignatureFormatter(FormioFormatter): - def format(self, component: Component, value: str) -> str: - text = _("signature added") - if not self.as_html: - return text - - assert value.startswith( - "data:image/" - ), "Expected 'data:' URI with image mime type" - - # max-width is required for e-mail styling where it may overflow a table cell - return format_html( - """{alt}""", - src=value, - alt=text, - ) - - -@register("map") -class MapFormatter(FormioFormatter): - def format(self, component: Component, value: List[float]) -> str: - # use a comma here since its a single data element - return join_mapped(value, separator=", ") diff --git a/src/openforms/formio/formatters/registry.py b/src/openforms/formio/formatters/registry.py deleted file mode 100644 index 0cb7b16cd7..0000000000 --- a/src/openforms/formio/formatters/registry.py +++ /dev/null @@ -1,22 +0,0 @@ -from typing import Any - -from openforms.plugins.registry import BaseRegistry - -from ..typing import Component - - -class Registry(BaseRegistry): - """ - A registry for the FormIO formatters. - """ - - def format(self, info: Component, value: Any, as_html=False): - formatter = ( - register[info["type"]] if info["type"] in register else register["default"] - ) - return formatter(info, value, as_html=as_html) - - -# Sentinel to provide the default registry. You can easily instantiate another -# :class:`Registry` object to use as dependency injection in tests. -register = Registry() From e830fd17a5e20d63aa3e6c95af32c43788f02cf1 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Wed, 23 Nov 2022 17:05:36 +0100 Subject: [PATCH 04/16] :wastebasket: Deprecate format_value in favour of higher-level public service export This moves it up into the openforms.formio.service module rather than in the formatters subpackage. --- src/openforms/emails/confirmation_emails.py | 2 +- src/openforms/formio/formatters/service.py | 17 ++++++++++------- .../formatters/tests/test_default_formatters.py | 2 +- .../formio/formatters/tests/test_issues.py | 2 +- .../formio/formatters/tests/test_kitchensink.py | 2 +- src/openforms/formio/rendering/nodes.py | 4 ++-- src/openforms/formio/service.py | 8 +++++++- src/openforms/submissions/models/submission.py | 3 ++- 8 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/openforms/emails/confirmation_emails.py b/src/openforms/emails/confirmation_emails.py index 645f3fac65..68977136ef 100644 --- a/src/openforms/emails/confirmation_emails.py +++ b/src/openforms/emails/confirmation_emails.py @@ -47,7 +47,7 @@ def get_confirmation_email_context_data(submission: "Submission") -> Dict[str, A # starting with underscores are blocked by the Django template engine. "_submission": submission, "_form": submission.form, # should be the same as self.form - # TODO: this should use the :func:`openforms.formio.formatters.service.format_value` + # TODO: this should use the :func:`openforms.formio.service.format_value` # but be keyed by component.key instead of the label, which # submission.get_printable_data did. **submission.data, diff --git a/src/openforms/formio/formatters/service.py b/src/openforms/formio/formatters/service.py index 8251a112cc..3bc22580f4 100644 --- a/src/openforms/formio/formatters/service.py +++ b/src/openforms/formio/formatters/service.py @@ -1,12 +1,15 @@ -from typing import Any +import warnings -from ..registry import register -from ..typing import Component +from ..service import format_value as _format_value from .printable import filter_printable -__all__ = ["format_value", "filter_printable"] +__all__ = ["filter_printable"] -# TODO: move to parent level service module and deprecate this one. -def format_value(component: Component, value: Any, as_html=False): - return register.format(component, value, as_html=as_html) +def format_value(*args, **kwargs): + warnings.warn( + "`openforms.formio.formatters.service.format_value` is deprecated in favour of " + "`openforms.formio.service.format_value`. Please update your references.", + DeprecationWarning, + ) + return _format_value(*args, **kwargs) diff --git a/src/openforms/formio/formatters/tests/test_default_formatters.py b/src/openforms/formio/formatters/tests/test_default_formatters.py index b061902f47..26581d2fd9 100644 --- a/src/openforms/formio/formatters/tests/test_default_formatters.py +++ b/src/openforms/formio/formatters/tests/test_default_formatters.py @@ -1,7 +1,7 @@ from django.test import TestCase from django.utils.translation import gettext_lazy as _ -from ..service import format_value +from ...service import format_value from .utils import load_json diff --git a/src/openforms/formio/formatters/tests/test_issues.py b/src/openforms/formio/formatters/tests/test_issues.py index 7829755fee..7a03c9fdc1 100644 --- a/src/openforms/formio/formatters/tests/test_issues.py +++ b/src/openforms/formio/formatters/tests/test_issues.py @@ -1,6 +1,6 @@ from django.test import TestCase -from openforms.formio.formatters.service import format_value +from ...service import format_value class IssuesTestCase(TestCase): diff --git a/src/openforms/formio/formatters/tests/test_kitchensink.py b/src/openforms/formio/formatters/tests/test_kitchensink.py index f7feea146b..d58ee6fe11 100644 --- a/src/openforms/formio/formatters/tests/test_kitchensink.py +++ b/src/openforms/formio/formatters/tests/test_kitchensink.py @@ -5,9 +5,9 @@ SubmissionFileAttachmentFactory, ) +from ...service import format_value from ...utils import iter_components from ..printable import filter_printable -from ..service import format_value from .mixins import BaseFormatterTestCase from .utils import load_json diff --git a/src/openforms/formio/rendering/nodes.py b/src/openforms/formio/rendering/nodes.py index 5cda34046b..00f31ecf79 100644 --- a/src/openforms/formio/rendering/nodes.py +++ b/src/openforms/formio/rendering/nodes.py @@ -7,7 +7,7 @@ from openforms.submissions.rendering.base import Node from openforms.submissions.rendering.constants import RenderModes -from ..formatters.service import format_value +from ..service import format_value from ..typing import Component from ..utils import is_layout_component, is_visible_in_frontend, iter_components @@ -189,7 +189,7 @@ def display_value(self) -> Union[str, Any]: Format the value according to the render mode and/or output content type. This applies the registry of Formio formatters to the value based on the - component type, using :func:`openforms.formio.formatter.service.format_value`. + component type, using :func:`openforms.formio.service.format_value`. """ # in export mode, expose the raw datatype if self.mode == RenderModes.export: diff --git a/src/openforms/formio/service.py b/src/openforms/formio/service.py index f9a38407d1..7f284ff108 100644 --- a/src/openforms/formio/service.py +++ b/src/openforms/formio/service.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Any, Optional from django.urls import reverse @@ -14,6 +14,7 @@ from .datastructures import FormioConfigurationWrapper from .dynamic_config.service import apply_dynamic_configuration from .normalization import normalize_value_for_component +from .registry import register from .typing import Component from .utils import iter_components from .variables import inject_variables @@ -24,11 +25,16 @@ "normalize_value_for_component", "iter_components", "inject_variables", + "format_value", ] from ..config.models import GlobalConfiguration +def format_value(component: Component, value: Any, as_html: bool = False): + return register.format(component, value, as_html=as_html) + + # TODO: it might be beneficial to memoize this function if it runs multiple times in # the context of the same request @elasticapm.capture_span(span_type="app.formio") diff --git a/src/openforms/submissions/models/submission.py b/src/openforms/submissions/models/submission.py index 169bfb4a71..2526d8325b 100644 --- a/src/openforms/submissions/models/submission.py +++ b/src/openforms/submissions/models/submission.py @@ -15,7 +15,6 @@ from openforms.config.models import GlobalConfiguration from openforms.formio.datastructures import FormioConfigurationWrapper -from openforms.formio.formatters.service import filter_printable from openforms.forms.models import FormStep from openforms.payments.constants import PaymentStatus from openforms.typing import JSONObject @@ -508,6 +507,8 @@ def get_last_completed_step(self) -> Optional["SubmissionStep"]: return submission_state.get_last_completed_step() def get_ordered_data_with_component_type(self) -> OrderedDict: + from openforms.formio.formatters.service import filter_printable + ordered_data = OrderedDict() merged_data = self.get_merged_data() From 303f9f0080a0c72b94f94e4672d447c491df2cd6 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Wed, 23 Nov 2022 17:19:18 +0100 Subject: [PATCH 05/16] :recycle: [#1068] -- only import normalization from public service API --- src/openforms/submissions/api/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/openforms/submissions/api/validators.py b/src/openforms/submissions/api/validators.py index 3632e8de3c..ad30040f77 100644 --- a/src/openforms/submissions/api/validators.py +++ b/src/openforms/submissions/api/validators.py @@ -3,7 +3,7 @@ from rest_framework import serializers from rest_framework.serializers import JSONField -from openforms.formio.normalization import normalize_value_for_component +from openforms.formio.service import normalize_value_for_component from openforms.forms.models import Form from ..exceptions import FormMaintenance From 2e0116b8d376bead4d8783853f6e6012ef263c32 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Wed, 23 Nov 2022 17:45:54 +0100 Subject: [PATCH 06/16] :recycle: [#1068] -- move data normalization to component plugins --- src/openforms/formio/components/custom.py | 29 +++++++++ src/openforms/formio/components/vanilla.py | 7 +++ src/openforms/formio/normalization.py | 72 ---------------------- src/openforms/formio/registry.py | 13 +++- src/openforms/formio/service.py | 9 ++- 5 files changed, 56 insertions(+), 74 deletions(-) delete mode 100644 src/openforms/formio/normalization.py diff --git a/src/openforms/formio/components/custom.py b/src/openforms/formio/components/custom.py index ed95c8de2f..528e353735 100644 --- a/src/openforms/formio/components/custom.py +++ b/src/openforms/formio/components/custom.py @@ -1,7 +1,36 @@ +import logging + from ..formatters.custom import MapFormatter +from ..formatters.formio import TextFieldFormatter from ..registry import BasePlugin, register +from ..typing import Component +from ..utils import conform_to_mask + +logger = logging.getLogger(__name__) @register("map") class Map(BasePlugin): formatter = MapFormatter + + +@register("postcode") +class Postcode(BasePlugin): + formatter = TextFieldFormatter + + @staticmethod + def normalizer(component: Component, value: str) -> str: + if not value: + return value + + input_mask = component.get("inputMask") + if not input_mask: + return value + + try: + return conform_to_mask(value, input_mask) + except ValueError: + logger.warning( + "Could not conform value '%s' to input mask '%s', returning original value." + ) + return value diff --git a/src/openforms/formio/components/vanilla.py b/src/openforms/formio/components/vanilla.py index 3253023f7b..3de8f0f71b 100644 --- a/src/openforms/formio/components/vanilla.py +++ b/src/openforms/formio/components/vanilla.py @@ -4,6 +4,8 @@ Custom component types (defined by us or third parties) need to be organized in the adjacent custom.py module. """ +from openforms.utils.date import format_date_value + from ..formatters.formio import ( CheckboxFormatter, CurrencyFormatter, @@ -23,6 +25,7 @@ TimeFormatter, ) from ..registry import BasePlugin, register +from ..typing import Component @register("default") @@ -48,6 +51,10 @@ class Email(BasePlugin): class Date(BasePlugin): formatter = DateFormatter + @staticmethod + def normalizer(component: Component, value: str) -> str: + return format_date_value(value) + @register("time") class Time(BasePlugin): diff --git a/src/openforms/formio/normalization.py b/src/openforms/formio/normalization.py deleted file mode 100644 index 5e95ae1abc..0000000000 --- a/src/openforms/formio/normalization.py +++ /dev/null @@ -1,72 +0,0 @@ -import logging -from abc import abstractmethod -from typing import Any - -from openforms.plugins.plugin import AbstractBasePlugin -from openforms.plugins.registry import BaseRegistry -from openforms.utils.date import format_date_value - -from .typing import Component -from .utils import conform_to_mask - -__all__ = ["normalize_value_for_component", "register", "Normalizer"] - -logger = logging.getLogger(__name__) - - -def normalize_value_for_component(component: Component, value: Any) -> Any: - """ - Given a value (actual or default value) and the component, apply the component- - specific normalization. - """ - if (component_type := component.get("type")) not in register: - return value - normalizer = register[component_type] - return normalizer(component, value) - - -class Registry(BaseRegistry): - """ - A registry for FormIO normalization functions. - """ - - pass - - -class Normalizer(AbstractBasePlugin): - def __call__(self, component: Component, value: Any) -> Any: - return self.normalize(component, value) - - @abstractmethod - def normalize(self, component: Component, value: Any) -> Any: - raise NotImplementedError # pragma: nocover - - -# Sentinel to provide the default registry. You an easily instantiate another -# :class:`Registry` object to use as dependency injection in tests. -register = Registry() - - -@register("postcode") -class PostalCodeNormalizer(Normalizer): - def normalize(self, component: Component, value: str) -> str: - if not value: - return value - - input_mask = component.get("inputMask") - if not input_mask: - return value - - try: - return conform_to_mask(value, input_mask) - except ValueError: - logger.warning( - "Could not conform value '%s' to input mask '%s', returning original value." - ) - return value - - -@register("date") -class DateNormalizer(Normalizer): - def normalize(self, component: Component, value: str) -> str: - return format_date_value(value) diff --git a/src/openforms/formio/registry.py b/src/openforms/formio/registry.py index 397ce4442a..58010dace3 100644 --- a/src/openforms/formio/registry.py +++ b/src/openforms/formio/registry.py @@ -32,6 +32,11 @@ def __call__(self, component: Component, value: Any) -> str: ... +class NormalizerProtocol(Protocol): + def __call__(self, component: Component, value: Any) -> Any: + ... + + class BasePlugin(AbstractBasePlugin): """ Base class for Formio component plugins. @@ -46,6 +51,7 @@ class BasePlugin(AbstractBasePlugin): Formatter (class) implementation, used by :meth:`openforms.formio.registry.ComponentRegistry.format`. """ + normalizer: None | NormalizerProtocol = None @staticmethod def mutate(component: Component, data: DataMapping) -> None: # pragma: nocover @@ -59,7 +65,12 @@ def normalize(self, component: Component, value: Any) -> Any: """ Given a value from any source, normalize it according to the component rules. """ - raise NotImplementedError() + if (component_type := component["type"]) not in self: + return value + normalizer = self[component_type].normalizer + if normalizer is None: + return value + return normalizer(component, value) def format(self, component: Component, value: Any, as_html=False): """ diff --git a/src/openforms/formio/service.py b/src/openforms/formio/service.py index 7f284ff108..81705c1c5f 100644 --- a/src/openforms/formio/service.py +++ b/src/openforms/formio/service.py @@ -13,7 +13,6 @@ from .datastructures import FormioConfigurationWrapper from .dynamic_config.service import apply_dynamic_configuration -from .normalization import normalize_value_for_component from .registry import register from .typing import Component from .utils import iter_components @@ -35,6 +34,14 @@ def format_value(component: Component, value: Any, as_html: bool = False): return register.format(component, value, as_html=as_html) +def normalize_value_for_component(component: Component, value: Any) -> Any: + """ + Given a value (actual or default value) and the component, apply the component- + specific normalization. + """ + return register.normalize(component, value) + + # TODO: it might be beneficial to memoize this function if it runs multiple times in # the context of the same request @elasticapm.capture_span(span_type="app.formio") From 9e312b0b661dbdd8e9e59e345c4ae5544ecbfa89 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 13:13:03 +0100 Subject: [PATCH 07/16] :recycle: [#1068] -- bring dynamic config mutations into main component/registry This deletes the dynamic config-specific registry, while mostly keeping the public API intact. --- src/openforms/conf/base.py | 1 - src/openforms/formio/components/custom.py | 26 ++- src/openforms/formio/components/vanilla.py | 13 -- .../formio/dynamic_config/__init__.py | 21 +++ src/openforms/formio/dynamic_config/apps.py | 9 - src/openforms/formio/dynamic_config/date.py | 172 ++++++++---------- .../formio/dynamic_config/registry.py | 40 ---- .../formio/dynamic_config/service.py | 24 --- src/openforms/formio/formatters/custom.py | 8 + src/openforms/formio/formatters/formio.py | 5 - src/openforms/formio/registry.py | 16 +- src/openforms/formio/service.py | 2 +- 12 files changed, 147 insertions(+), 190 deletions(-) delete mode 100644 src/openforms/formio/dynamic_config/apps.py delete mode 100644 src/openforms/formio/dynamic_config/registry.py delete mode 100644 src/openforms/formio/dynamic_config/service.py diff --git a/src/openforms/conf/base.py b/src/openforms/conf/base.py index 115ef07aa7..aca410fe9a 100644 --- a/src/openforms/conf/base.py +++ b/src/openforms/conf/base.py @@ -193,7 +193,6 @@ "openforms.config", "openforms.emails", "openforms.formio", - "openforms.formio.dynamic_config", "openforms.formio.rendering", "openforms.forms", "openforms.multidomain", diff --git a/src/openforms/formio/components/custom.py b/src/openforms/formio/components/custom.py index 528e353735..ab660ecd6c 100644 --- a/src/openforms/formio/components/custom.py +++ b/src/openforms/formio/components/custom.py @@ -1,6 +1,10 @@ import logging -from ..formatters.custom import MapFormatter +from openforms.typing import DataMapping +from openforms.utils.date import format_date_value + +from ..dynamic_config.date import FormioDateComponent, mutate as mutate_date +from ..formatters.custom import DateFormatter, MapFormatter from ..formatters.formio import TextFieldFormatter from ..registry import BasePlugin, register from ..typing import Component @@ -9,6 +13,26 @@ logger = logging.getLogger(__name__) +@register("date") +class Date(BasePlugin): + formatter = DateFormatter + + @staticmethod + def normalizer(component: FormioDateComponent, value: str) -> str: + return format_date_value(value) + + def mutate_config_dynamically( + self, component: FormioDateComponent, data: DataMapping + ) -> None: + """ + Implement the behaviour for our custom date component options. + + In the JS, this component type inherits from Formio datetime component. See + ``src/openforms/js/components/form/date.js`` for the various configurable options. + """ + mutate_date(component, data) + + @register("map") class Map(BasePlugin): formatter = MapFormatter diff --git a/src/openforms/formio/components/vanilla.py b/src/openforms/formio/components/vanilla.py index 3de8f0f71b..6c2094a6a2 100644 --- a/src/openforms/formio/components/vanilla.py +++ b/src/openforms/formio/components/vanilla.py @@ -4,12 +4,9 @@ Custom component types (defined by us or third parties) need to be organized in the adjacent custom.py module. """ -from openforms.utils.date import format_date_value - from ..formatters.formio import ( CheckboxFormatter, CurrencyFormatter, - DateFormatter, DefaultFormatter, EmailFormatter, FileFormatter, @@ -25,7 +22,6 @@ TimeFormatter, ) from ..registry import BasePlugin, register -from ..typing import Component @register("default") @@ -47,15 +43,6 @@ class Email(BasePlugin): formatter = EmailFormatter -@register("date") -class Date(BasePlugin): - formatter = DateFormatter - - @staticmethod - def normalizer(component: Component, value: str) -> str: - return format_date_value(value) - - @register("time") class Time(BasePlugin): formatter = TimeFormatter diff --git a/src/openforms/formio/dynamic_config/__init__.py b/src/openforms/formio/dynamic_config/__init__.py index 0db499f65a..3d98722f86 100644 --- a/src/openforms/formio/dynamic_config/__init__.py +++ b/src/openforms/formio/dynamic_config/__init__.py @@ -3,3 +3,24 @@ This will eventually replace ``openforms.forms.custom_field_types``. """ +from typing import Optional + +from openforms.typing import DataMapping + +from ..datastructures import FormioConfigurationWrapper +from ..registry import register + +__all__ = ["apply_dynamic_configuration"] + + +def apply_dynamic_configuration( + configuration_wrapper: FormioConfigurationWrapper, + data: Optional[DataMapping] = None, +) -> FormioConfigurationWrapper: + """ + Loop over the formio configuration and mutate components in place. + """ + data = data or {} # normalize + for component in configuration_wrapper: + register.update_config(component, data=data) + return configuration_wrapper diff --git a/src/openforms/formio/dynamic_config/apps.py b/src/openforms/formio/dynamic_config/apps.py deleted file mode 100644 index c7e4022398..0000000000 --- a/src/openforms/formio/dynamic_config/apps.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.apps import AppConfig - - -class FormioDynamicConfigApp(AppConfig): - name = "openforms.formio.dynamic_config" - label = "formio_dynamic_config" - - def ready(self): - from . import date # noqa - register the plugins diff --git a/src/openforms/formio/dynamic_config/date.py b/src/openforms/formio/dynamic_config/date.py index 40839a049a..e410d036cb 100644 --- a/src/openforms/formio/dynamic_config/date.py +++ b/src/openforms/formio/dynamic_config/date.py @@ -3,7 +3,6 @@ from typing import Literal, Optional, TypedDict, Union, cast from django.utils import timezone -from django.utils.translation import gettext_lazy as _ from dateutil.relativedelta import relativedelta from glom import assign, glom @@ -11,7 +10,6 @@ from openforms.typing import DataMapping from ..typing import Component -from .registry import BasePlugin, register NOW_VARIABLE = "now" @@ -45,99 +43,87 @@ class FormioDateComponent(Component): datePicker: Optional[DatePickerConfig] -@register("date") -class DateComponent(BasePlugin): - """ - Implement behaviour for our custom date component - - In the JS, this component type inherits from Formio datetime component. See - ``src/openforms/js/components/form/date.js`` for the various configurable options. - """ - - verbose_name = _("Date component") - - @classmethod - def mutate(cls, component: FormioDateComponent, data: DataMapping) -> None: - for key in ("minDate", "maxDate"): - config = cast( - DateConstraintConfiguration, - glom(component, f"openForms.{key}", default=None), - ) - if config is None: - continue - - if (mode := config.get("mode")) is None: - continue - - # formio has set the datePicker.{key} value - if mode == "fixedValue": - continue - - config = cls.normalize_config(config) - value = cls.calculate_delta(config, data) - if value and isinstance(value, datetime): - value = value.isoformat() - assign(component, f"datePicker.{key}", value, missing=dict) - - @staticmethod - def normalize_config( - config: DateConstraintConfiguration, - ) -> DateConstraintConfiguration: - mode = config["mode"] - assert mode in ("future", "past", "relativeToVariable") - if mode == "relativeToVariable": - return config - - # mode is now future or past -> convert that to a relative delta config - config["mode"] = "relativeToVariable" - config["variable"] = NOW_VARIABLE - include_today = cast(bool, glom(config, "includeToday", default=False)) - config["operator"] = "add" if mode == "future" else "subtract" - - delta = cast( - DateConstraintDelta, - { - "years": 0, - "months": 0, - "days": 0 if include_today else 1, - }, +def mutate(component: FormioDateComponent, data: DataMapping) -> None: + for key in ("minDate", "maxDate"): + config = cast( + DateConstraintConfiguration, + glom(component, f"openForms.{key}", default=None), ) - config["delta"] = delta - return config + if config is None: + continue - @staticmethod - def calculate_delta( - config: DateConstraintConfiguration, - data: DataMapping, - ) -> Optional[datetime]: - assert config["mode"] == "relativeToVariable" + if (mode := config.get("mode")) is None: + continue - base_value = cast( - Optional[Union[datetime, str]], - glom(data, config["variable"], default=None), - ) - # can't do calculations on values that don't exist or are empty - if not base_value: - return None - - # if it's not empty-ish, it's a datetime - base_value = cast(datetime, base_value) - - assert ( - base_value.tzinfo is not None - ), "Expected the input variable to be timezone aware!" - base_date = timezone.localtime(value=base_value).date() - - delta = relativedelta( - years=cast(int, glom(config, "delta.years", default=None) or 0), - months=cast(int, glom(config, "delta.months", default=None) or 0), - days=cast(int, glom(config, "delta.days", default=None) or 0), - ) + # formio has set the datePicker.{key} value + if mode == "fixedValue": + continue - add_or_subtract = glom(config, "operator", default="add") - func = operator.add if add_or_subtract == "add" else operator.sub - value = func(base_date, delta) + config = normalize_config(config) + value = calculate_delta(config, data) + if value and isinstance(value, datetime): + value = value.isoformat() + assign(component, f"datePicker.{key}", value, missing=dict) + + +def normalize_config( + config: DateConstraintConfiguration, +) -> DateConstraintConfiguration: + mode = config["mode"] + assert mode in ("future", "past", "relativeToVariable") + if mode == "relativeToVariable": + return config - # convert to datetime at midnight for the date in the local timezone - naive_value = datetime.combine(value, time.min) - return timezone.make_aware(naive_value) + # mode is now future or past -> convert that to a relative delta config + config["mode"] = "relativeToVariable" + config["variable"] = NOW_VARIABLE + include_today = cast(bool, glom(config, "includeToday", default=False)) + config["operator"] = "add" if mode == "future" else "subtract" + + delta = cast( + DateConstraintDelta, + { + "years": 0, + "months": 0, + "days": 0 if include_today else 1, + }, + ) + config["delta"] = delta + return config + + +def calculate_delta( + config: DateConstraintConfiguration, + data: DataMapping, +) -> Optional[datetime]: + assert config["mode"] == "relativeToVariable" + + base_value = cast( + Optional[Union[datetime, str]], + glom(data, config["variable"], default=None), + ) + # can't do calculations on values that don't exist or are empty + if not base_value: + return None + + # if it's not empty-ish, it's a datetime + base_value = cast(datetime, base_value) + + assert ( + base_value.tzinfo is not None + ), "Expected the input variable to be timezone aware!" + base_date = timezone.localtime(value=base_value).date() + + delta = relativedelta( + years=cast(int, glom(config, "delta.years", default=None) or 0), + months=cast(int, glom(config, "delta.months", default=None) or 0), + days=cast(int, glom(config, "delta.days", default=None) or 0), + ) + + add_or_subtract = glom(config, "operator", default="add") + func = operator.add if add_or_subtract == "add" else operator.sub + value = func(base_date, delta) + + # convert to datetime at midnight for the date in the local timezone + naive_value = datetime.combine(value, time.min) + return timezone.make_aware(naive_value) diff --git a/src/openforms/formio/dynamic_config/registry.py b/src/openforms/formio/dynamic_config/registry.py deleted file mode 100644 index 1b4f0fdcf1..0000000000 --- a/src/openforms/formio/dynamic_config/registry.py +++ /dev/null @@ -1,40 +0,0 @@ -from typing import Optional - -from openforms.plugins.plugin import AbstractBasePlugin -from openforms.plugins.registry import BaseRegistry -from openforms.typing import DataMapping - -from ..typing import Component - - -class BasePlugin(AbstractBasePlugin): - is_enabled: bool = True - - @staticmethod - def mutate(component: Component, data: DataMapping) -> None: # pragma: nocover - ... - - -class Registry(BaseRegistry): - """ - A registry for the FormIO formatters. - """ - - def update_config( - self, component: Component, data: Optional[DataMapping] = None - ) -> None: - """ - Mutate the component configuration in place. - """ - # if there is no plugin registered for the component, return the input - if (component_type := component["type"]) not in register: - return - - # invoke plugin if exists - plugin = self[component_type] - plugin.mutate(component, data) - - -# Sentinel to provide the default registry. You can easily instantiate another -# :class:`Registry` object to use as dependency injection in tests. -register = Registry() diff --git a/src/openforms/formio/dynamic_config/service.py b/src/openforms/formio/dynamic_config/service.py deleted file mode 100644 index 32599a1ca7..0000000000 --- a/src/openforms/formio/dynamic_config/service.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Public API for dynamic configuration. -""" -from typing import Optional - -from openforms.typing import DataMapping - -from ..datastructures import FormioConfigurationWrapper -from .registry import register - -__all__ = ["apply_dynamic_configuration"] - - -def apply_dynamic_configuration( - configuration_wrapper: FormioConfigurationWrapper, - data: Optional[DataMapping] = None, -) -> FormioConfigurationWrapper: - """ - Loop over the formio configuration and mutate components in place. - """ - data = data or {} # normalize - for component in configuration_wrapper: - register.update_config(component, data=data) - return configuration_wrapper diff --git a/src/openforms/formio/formatters/custom.py b/src/openforms/formio/formatters/custom.py index 7732156ec3..2d46e69243 100644 --- a/src/openforms/formio/formatters/custom.py +++ b/src/openforms/formio/formatters/custom.py @@ -1,8 +1,16 @@ # TODO implement: iban, bsn, postcode, licenseplate, npFamilyMembers, cosign +from django.template.defaultfilters import date as fmt_date +from django.utils.dateparse import parse_date + from ..typing import Component from .base import FormatterBase +class DateFormatter(FormatterBase): + def format(self, component: Component, value: str) -> str: + return fmt_date(parse_date(value)) + + class MapFormatter(FormatterBase): def format(self, component: Component, value: list[float]) -> str: # use a comma here since its a single data element diff --git a/src/openforms/formio/formatters/formio.py b/src/openforms/formio/formatters/formio.py index 970795f873..d77a8dfb04 100644 --- a/src/openforms/formio/formatters/formio.py +++ b/src/openforms/formio/formatters/formio.py @@ -55,11 +55,6 @@ def format(self, component: Component, value: str) -> str: return str(value) -class DateFormatter(FormatterBase): - def format(self, component: Component, value: str) -> str: - return fmt_date(parse_date(value)) - - class TimeFormatter(FormatterBase): def format(self, component: Component, value: str) -> str: return fmt_time(parse_time(value)) diff --git a/src/openforms/formio/registry.py b/src/openforms/formio/registry.py index 58010dace3..9648d7e1b1 100644 --- a/src/openforms/formio/registry.py +++ b/src/openforms/formio/registry.py @@ -13,6 +13,8 @@ """ from typing import TYPE_CHECKING, Any, Protocol +from django.utils.translation import gettext as _ + from openforms.plugins.plugin import AbstractBasePlugin from openforms.plugins.registry import BaseRegistry from openforms.typing import DataMapping @@ -52,9 +54,17 @@ class BasePlugin(AbstractBasePlugin): :meth:`openforms.formio.registry.ComponentRegistry.format`. """ normalizer: None | NormalizerProtocol = None + """ + Specify the normalizer callable to use for value normalization. + """ + + @property + def verbose_name(self): + return _("{type} component").format(type=self.identifier.capitalize()) - @staticmethod - def mutate(component: Component, data: DataMapping) -> None: # pragma: nocover + def mutate_config_dynamically( + self, component: Component, data: DataMapping + ) -> None: # pragma: nocover ... @@ -102,7 +112,7 @@ def update_config( # invoke plugin if exists plugin = self[component_type] - plugin.mutate(component, data) + plugin.mutate_config_dynamically(component, data) def handle_custom_types( self, diff --git a/src/openforms/formio/service.py b/src/openforms/formio/service.py index 81705c1c5f..422566b31d 100644 --- a/src/openforms/formio/service.py +++ b/src/openforms/formio/service.py @@ -12,7 +12,7 @@ from openforms.typing import DataMapping from .datastructures import FormioConfigurationWrapper -from .dynamic_config.service import apply_dynamic_configuration +from .dynamic_config import apply_dynamic_configuration from .registry import register from .typing import Component from .utils import iter_components From 79fd42e03a8b6391158e69f149b317c665a92634 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 15:10:25 +0100 Subject: [PATCH 08/16] :recycle: [#1068] -- refactor 'update for request' component processing pipeline Instead of using a fixed pipeline, the concerns are now moved to the component level that can opt-in to request-specific component rewriting. This should also lead to slightly more optimized code execution since we're looping over a caching datastructure rather than looping over the entire component tree over and over again. --- src/openforms/formio/components/vanilla.py | 56 +++++++++++++++ .../formio/dynamic_config/__init__.py | 26 ++++++- src/openforms/formio/registry.py | 26 +++++++ src/openforms/formio/service.py | 70 +++++-------------- src/openforms/formio/tests/test_service.py | 6 +- .../forms/api/serializers/form_definition.py | 15 ++-- .../tests/test_get_submission_step.py | 33 +++++++++ 7 files changed, 169 insertions(+), 63 deletions(-) diff --git a/src/openforms/formio/components/vanilla.py b/src/openforms/formio/components/vanilla.py index 6c2094a6a2..b22c3c92d4 100644 --- a/src/openforms/formio/components/vanilla.py +++ b/src/openforms/formio/components/vanilla.py @@ -4,6 +4,15 @@ Custom component types (defined by us or third parties) need to be organized in the adjacent custom.py module. """ +from typing import Literal + +from rest_framework.request import Request +from rest_framework.reverse import reverse + +from csp_post_processor import post_process_html +from openforms.config.models import GlobalConfiguration +from openforms.utils.urls import build_absolute_uri + from ..formatters.formio import ( CheckboxFormatter, CurrencyFormatter, @@ -22,6 +31,7 @@ TimeFormatter, ) from ..registry import BasePlugin, register +from ..typing import Component @register("default") @@ -53,10 +63,28 @@ class PhoneNumber(BasePlugin): formatter = PhoneNumberFormatter +class FileComponent(Component): + storage: Literal["url"] + url: str + useConfigFiletypes: bool + filePattern: str + + @register("file") class File(BasePlugin): formatter = FileFormatter + @staticmethod + def rewrite_for_request(component: FileComponent, request: Request): + # write the upload endpoint information + upload_endpoint = reverse("api:formio:temporary-file-upload") + component["url"] = build_absolute_uri(upload_endpoint, request=request) + + # check if we need to apply "filePattern" modifications + if component.get("useConfigFiletypes", False): + config = GlobalConfiguration.get_solo() + component["filePattern"] = ",".join(config.form_upload_default_file_types) + @register("textarea") class TextArea(BasePlugin): @@ -101,3 +129,31 @@ class Radio(BasePlugin): @register("signature") class Signature(BasePlugin): formatter = SignatureFormatter + + +class ContentComponent(Component): + html: str + + +@register("content") +class Content(BasePlugin): + """ + Formio's WYSIWYG component. + """ + + # not really relevant as content components don't have values + formatter = DefaultFormatter + + @staticmethod + def rewrite_for_request(component: ContentComponent, request: Request): + """ + Ensure that the inline styles are made compatible with Content-Security-Policy. + + .. note:: we apply Bleach and a CSS declaration allowlist as part of the + post-processor because content components are not purely "trusted" content + from form-designers, but can contain malicious user input if the form + designer uses variables inside the HTML. The form submission data is passed + as template context to these HTML blobs, posing a potential injection + security risk. + """ + component["html"] = post_process_html(component["html"], request) diff --git a/src/openforms/formio/dynamic_config/__init__.py b/src/openforms/formio/dynamic_config/__init__.py index 3d98722f86..e405f01f84 100644 --- a/src/openforms/formio/dynamic_config/__init__.py +++ b/src/openforms/formio/dynamic_config/__init__.py @@ -5,22 +5,44 @@ """ from typing import Optional +from rest_framework.request import Request + from openforms.typing import DataMapping from ..datastructures import FormioConfigurationWrapper from ..registry import register -__all__ = ["apply_dynamic_configuration"] +__all__ = ["rewrite_formio_components", "rewrite_formio_components_for_request"] -def apply_dynamic_configuration( +def rewrite_formio_components( configuration_wrapper: FormioConfigurationWrapper, data: Optional[DataMapping] = None, ) -> FormioConfigurationWrapper: """ Loop over the formio configuration and mutate components in place. + + :arg configuration_wrapper: Container object holding the Formio form configuration + to be updated (if applicable). The rewriting loops over all components one-by-one + and applies the changes. + :arg data: key-value mapping of variable name to variable value. If a submission + context is available, the variables of the submission are included here. + :arg request: The HTTP request received by the API view when rewriting is done as + part of an HTTP request-response cycle. This is ``None`` otherwise (e.g. in + background tasks). """ data = data or {} # normalize for component in configuration_wrapper: register.update_config(component, data=data) return configuration_wrapper + + +def rewrite_formio_components_for_request( + configuration_wrapper: FormioConfigurationWrapper, request: Request +) -> FormioConfigurationWrapper: + """ + Loop over the formio configuration and inject request-specific configuration. + """ + for component in configuration_wrapper: + register.update_config_for_request(component, request=request) + return configuration_wrapper diff --git a/src/openforms/formio/registry.py b/src/openforms/formio/registry.py index 9648d7e1b1..b0d8de444c 100644 --- a/src/openforms/formio/registry.py +++ b/src/openforms/formio/registry.py @@ -15,6 +15,8 @@ from django.utils.translation import gettext as _ +from rest_framework.request import Request + from openforms.plugins.plugin import AbstractBasePlugin from openforms.plugins.registry import BaseRegistry from openforms.typing import DataMapping @@ -39,6 +41,11 @@ def __call__(self, component: Component, value: Any) -> Any: ... +class RewriterForRequestProtocol(Protocol): + def __call__(self, component: Component, request: Request) -> None: + ... + + class BasePlugin(AbstractBasePlugin): """ Base class for Formio component plugins. @@ -57,6 +64,10 @@ class BasePlugin(AbstractBasePlugin): """ Specify the normalizer callable to use for value normalization. """ + rewrite_for_request: None | RewriterForRequestProtocol = None + """ + Callback to invoke to rewrite plugin configuration for a given HTTP request. + """ @property def verbose_name(self): @@ -114,6 +125,21 @@ def update_config( plugin = self[component_type] plugin.mutate_config_dynamically(component, data) + def update_config_for_request(self, component: Component, request: Request) -> None: + """ + Mutate the component in place for the given request context. + """ + # if there is no plugin registered for the component, return the input + if (component_type := component["type"]) not in register: + return + + # invoke plugin if exists + rewriter = self[component_type].rewrite_for_request + if rewriter is None: + return + + rewriter(component, request) + def handle_custom_types( self, configuration: FormioConfigurationWrapper, diff --git a/src/openforms/formio/service.py b/src/openforms/formio/service.py index 422566b31d..4a6a10ca29 100644 --- a/src/openforms/formio/service.py +++ b/src/openforms/formio/service.py @@ -1,18 +1,28 @@ -from typing import Any, Optional +""" +Expose the public openforms.formio Python API. + +The service module exposes the functions/utilities that may be used by other Django +apps/packages: -from django.urls import reverse +* Try to keep this module stable and avoid breaking changes - extensions may rely on this! +* Keep it small! The actual implementation should be done in specialized subpackages or + submodules and only their 'public' API should be imported and used. +""" +from typing import Any, Optional import elasticapm from rest_framework.request import Request -from csp_post_processor import post_process_html from openforms.forms.custom_field_types import handle_custom_types from openforms.prefill import inject_prefill from openforms.submissions.models import Submission from openforms.typing import DataMapping from .datastructures import FormioConfigurationWrapper -from .dynamic_config import apply_dynamic_configuration +from .dynamic_config import ( + rewrite_formio_components, + rewrite_formio_components_for_request, +) from .registry import register from .typing import Component from .utils import iter_components @@ -20,15 +30,13 @@ __all__ = [ "get_dynamic_configuration", - "update_configuration_for_request", "normalize_value_for_component", "iter_components", "inject_variables", "format_value", + "rewrite_formio_components_for_request", ] -from ..config.models import GlobalConfiguration - def format_value(component: Component, value: Any, as_html: bool = False): return register.format(component, value, as_html=as_html) @@ -42,8 +50,6 @@ def normalize_value_for_component(component: Component, value: Any) -> Any: return register.normalize(component, value) -# TODO: it might be beneficial to memoize this function if it runs multiple times in -# the context of the same request @elasticapm.capture_span(span_type="app.formio") def get_dynamic_configuration( config_wrapper: FormioConfigurationWrapper, @@ -61,7 +67,8 @@ def get_dynamic_configuration( config_wrapper.configuration, request=request, submission=submission ) - apply_dynamic_configuration(config_wrapper, data=data) + rewrite_formio_components(config_wrapper, data=data) + # prefill is still 'special' even though it uses variables, as we specifically # set the `defaultValue` key to the resulting variable. # This *could* be refactored in the future by assigning a template expression to @@ -70,46 +77,3 @@ def get_dynamic_configuration( # as checkboxes/dropdowns/radios/... inject_prefill(config_wrapper, submission) return config_wrapper - - -def update_configuration_for_request( - config_wrapper: FormioConfigurationWrapper, request: Request -) -> None: - """ - Given a static Formio configuration, apply dynamic changes we always must do, like setting absolute urls. - - The configuration is modified in the context of the provided :arg:`submission`. - """ - - # TODO move this to openforms.formio.dynamic_config - pipeline = ( - update_urls_in_place, - update_default_file_types, - update_content_inline_csp, - ) - for component in config_wrapper: - for function in pipeline: - function(component, request=request) - - -def update_urls_in_place(component: Component, request: Request) -> None: - if component.get("type") == "file": - component["url"] = request.build_absolute_uri( - reverse("api:formio:temporary-file-upload") - ) - - -def update_default_file_types(component: Component, **kwargs) -> None: - if component.get("type") == "file" and component.get("useConfigFiletypes"): - config = GlobalConfiguration.get_solo() - component["filePattern"] = ",".join(config.form_upload_default_file_types) - - -def update_content_inline_csp(component: Component, request: Request) -> None: - if component.get("type") == "content": - """ - NOTE: we apply Bleach and a CSS declaration whitelist because content components are not purely "trusted" content from form-designers, - but can contain malicious user input if the form designer uses variables inside the HTML - (and the form submission data is passed as template context to those HTML blobs) - """ - component["html"] = post_process_html(component["html"], request) diff --git a/src/openforms/formio/tests/test_service.py b/src/openforms/formio/tests/test_service.py index 9b8d7e5008..a4169f3ec8 100644 --- a/src/openforms/formio/tests/test_service.py +++ b/src/openforms/formio/tests/test_service.py @@ -5,13 +5,13 @@ from openforms.formio.service import ( FormioConfigurationWrapper, - update_configuration_for_request, + rewrite_formio_components_for_request, ) class ServiceTestCase(TestCase): @patch("csp_post_processor.processor.get_html_id", return_value="1234") - def test_update_configuration_for_request(self, m): + def test_rewrite_formio_components_for_request(self, m): request = RequestFactory().get("/", HTTP_X_CSP_NONCE="dGVzdA==") configuration = { @@ -31,7 +31,7 @@ def test_update_configuration_for_request(self, m): }, ], } - update_configuration_for_request( + rewrite_formio_components_for_request( FormioConfigurationWrapper(configuration), request ) with self.subTest("temporary file upload url"): diff --git a/src/openforms/forms/api/serializers/form_definition.py b/src/openforms/forms/api/serializers/form_definition.py index 496b8eef72..6b6305ed63 100644 --- a/src/openforms/forms/api/serializers/form_definition.py +++ b/src/openforms/forms/api/serializers/form_definition.py @@ -5,7 +5,7 @@ from drf_spectacular.utils import extend_schema_field from rest_framework import serializers -from openforms.formio.service import update_configuration_for_request +from openforms.formio.service import rewrite_formio_components_for_request from ...models import Form, FormDefinition from ...validators import validate_form_definition_is_reusable @@ -50,10 +50,15 @@ def get_admin_url(self, obj: Form) -> str: class FormDefinitionSerializer(serializers.HyperlinkedModelSerializer): def to_representation(self, instance): representation = super().to_representation(instance=instance) - # set upload urls etc - # TODO: move this to openforms.formio.dynamic_config - update_configuration_for_request( - instance.configuration_wrapper, request=self.context["request"] + # Finalize formio component configuration with dynamic parts that depend on the + # HTTP request. Note that this is invoked through: + # 1. the :class:`openforms.submissions.api.serializers.SubmissionStepSerializer` + # for the dynamic formio configuration in the context of a submission. + # 2. The serializers/API endpoints of :module:`openforms.forms.api` for + # 'standalone' use/introspection. + rewrite_formio_components_for_request( + instance.configuration_wrapper, + request=self.context["request"], ) representation["configuration"] = instance.configuration_wrapper.configuration return representation diff --git a/src/openforms/submissions/tests/test_get_submission_step.py b/src/openforms/submissions/tests/test_get_submission_step.py index 80690320e6..201fb3fa2d 100644 --- a/src/openforms/submissions/tests/test_get_submission_step.py +++ b/src/openforms/submissions/tests/test_get_submission_step.py @@ -10,6 +10,8 @@ """ import uuid +from django.test import tag + from rest_framework import status from rest_framework.reverse import reverse from rest_framework.test import APITestCase @@ -187,6 +189,37 @@ def test_step_data_returned(self): {"someField": "data"}, ) + @tag("gh-1208", "gh-1068") + def test_dynamic_config_applied(self): + submission = SubmissionFactory.from_components( + [ + { + "type": "file", + "key": "file", + "storage": "url", + "url": "", # must be set dynamically + } + ] + ) + self._add_submission_to_session(submission) + step = submission.submissionstep_set.get() + url = reverse( + "api:submission-steps-detail", + kwargs={ + "submission_uuid": submission.uuid, + "step_uuid": step.form_step.uuid, + }, + ) + + response = self.client.get(url) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + data = response.json() + component = data["formStep"]["configuration"]["components"][0] + self.assertEqual(component["key"], "file") + self.assertEqual(component["type"], "file") + self.assertEqual(component["url"], "http://testserver/api/v2/formio/fileupload") + class GetSubmissionStepTests(SubmissionsMixin, APITestCase): @classmethod From 89c8a82525ddbd892e52a69bd0d23570298dcb50 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 15:39:30 +0100 Subject: [PATCH 09/16] :recycle: [#1068] -- removed need for request in custom field handlers First step towards refactoring how custom fields are handled, the request should not be needed as all required information is/should be present on the submission instance. --- .../custom_field_types/family_members.py | 14 ++++++-------- .../tests/test_family_members.py | 18 +++--------------- src/openforms/formio/service.py | 2 +- src/openforms/forms/custom_field_types.py | 4 +--- 4 files changed, 11 insertions(+), 27 deletions(-) diff --git a/src/openforms/custom_field_types/family_members.py b/src/openforms/custom_field_types/family_members.py index 1bc7f3b398..31c2b4f644 100644 --- a/src/openforms/custom_field_types/family_members.py +++ b/src/openforms/custom_field_types/family_members.py @@ -1,8 +1,7 @@ from typing import Any, Dict -from rest_framework.request import Request - -from openforms.authentication.constants import FORM_AUTH_SESSION_KEY, AuthAttribute +from openforms.authentication.constants import AuthAttribute +from openforms.formio.typing import Component from openforms.forms.custom_field_types import register from openforms.submissions.models import Submission @@ -14,18 +13,17 @@ @register("npFamilyMembers") def fill_out_family_members( - component: Dict[str, Any], request: Request, submission: Submission + component: Component, submission: Submission ) -> Dict[str, Any]: # Check authentication details - form_auth = request.session.get(FORM_AUTH_SESSION_KEY) - if not form_auth: + if not submission.is_authenticated: raise RuntimeError("No authenticated person!") - if form_auth.get("attribute") != AuthAttribute.bsn: + if submission.auth_info.attribute != AuthAttribute.bsn: raise RuntimeError("No BSN found in the authentication details.") - bsn = form_auth["value"] + bsn = submission.auth_info.value # Decide which API should be used to retrieve the children's data config = FamilyMembersTypeConfig.get_solo() diff --git a/src/openforms/custom_field_types/tests/test_family_members.py b/src/openforms/custom_field_types/tests/test_family_members.py index f2ba3aa438..4d3cfa115a 100644 --- a/src/openforms/custom_field_types/tests/test_family_members.py +++ b/src/openforms/custom_field_types/tests/test_family_members.py @@ -2,14 +2,12 @@ import os from unittest.mock import patch -from django.contrib.sessions.middleware import SessionMiddleware from django.template import Context, Template from django.test import TestCase -from django.test.client import RequestFactory import requests_mock -from openforms.authentication.constants import FORM_AUTH_SESSION_KEY, AuthAttribute +from openforms.authentication.constants import AuthAttribute from openforms.contrib.brp.models import BRPConfig from openforms.forms.custom_field_types import handle_custom_types from openforms.prefill.contrib.haalcentraal.tests.test_plugin import load_binary_mock @@ -31,15 +29,6 @@ class FamilyMembersCustomFieldTypeTest(TestCase): return_value=[("222333444", "Billy Doe"), ("333444555", "Jane Doe")], ) def test_get_values_for_custom_field(self, m): - request = RequestFactory().get("/") - middleware = SessionMiddleware() - middleware.process_request(request) - request.session[FORM_AUTH_SESSION_KEY] = { - "attribute": AuthAttribute.bsn, - "value": "111222333", - } - request.session.save() - configuration = { "display": "form", "components": [ @@ -52,6 +41,7 @@ def test_get_values_for_custom_field(self, m): ], } submission = SubmissionFactory.create( + auth_info__attribute=AuthAttribute.bsn, auth_info__value="111222333", form__generate_minimal_setup=True, form__formstep__form_definition__configuration=configuration, @@ -60,9 +50,7 @@ def test_get_values_for_custom_field(self, m): config.data_api = FamilyMembersDataAPIChoices.haal_centraal config.save() - rewritten_configuration = handle_custom_types( - configuration, request, submission - ) + rewritten_configuration = handle_custom_types(configuration, submission) rewritten_component = rewritten_configuration["components"][0] self.assertEqual("selectboxes", rewritten_component["type"]) diff --git a/src/openforms/formio/service.py b/src/openforms/formio/service.py index 4a6a10ca29..28d91bf47e 100644 --- a/src/openforms/formio/service.py +++ b/src/openforms/formio/service.py @@ -64,7 +64,7 @@ def get_dynamic_configuration( """ # TODO: see if we can make the config wrapper smart enough to deal with this config_wrapper.configuration = handle_custom_types( - config_wrapper.configuration, request=request, submission=submission + config_wrapper.configuration, submission=submission ) rewrite_formio_components(config_wrapper, data=data) diff --git a/src/openforms/forms/custom_field_types.py b/src/openforms/forms/custom_field_types.py index 11a6290e75..37899deb47 100644 --- a/src/openforms/forms/custom_field_types.py +++ b/src/openforms/forms/custom_field_types.py @@ -1,7 +1,6 @@ from typing import Any, Dict import elasticapm -from rest_framework.request import Request from openforms.submissions.models import Submission @@ -28,7 +27,6 @@ def unregister(custom_type: str): @elasticapm.capture_span(span_type="app.formio") def handle_custom_types( configuration: Dict[str, Any], - request: Request, submission: Submission, ) -> Dict[str, Any]: @@ -44,7 +42,7 @@ def handle_custom_types( # if there is a handler, invoke it handler = REGISTRY[type_key] - rewritten_components.append(handler(component, request, submission)) + rewritten_components.append(handler(component, submission)) return { "components": rewritten_components, From 19a4baa4d858072cbbf49b729d6402dc6b28ca11 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 16:21:25 +0100 Subject: [PATCH 10/16] :recycle: [#1068] -- refactor custom formio field types to use dynamic config approach Now it's just another flavour of mutating an existing component on the fly, with the added functionality of making some network calls to retrieve the relevant data. This essentially does away with the whole special treatment of custom field types. The use case of adding one or more components and rebuilding the component tree is removed, as there has not been any demand for this. We do keep in mind it may at some point be needed to inject components at other locations in the tree or do more advanced operations, but the current implementation does not sit in the way of that approach. Additionally, this fixes the bug where only top-level npFamilyMembers components could function correctly, which is now properly recursive for nested components. --- src/openforms/custom_field_types/__init__.py | 8 ++ src/openforms/custom_field_types/apps.py | 8 -- .../custom_field_types/family_members.py | 58 -------------- .../tests/test_family_members.py | 41 +++++----- src/openforms/formio/components/custom.py | 77 ++++++++++++++++++- .../formio/dynamic_config/__init__.py | 4 +- src/openforms/formio/registry.py | 29 ++----- src/openforms/formio/service.py | 8 +- src/openforms/forms/custom_field_types.py | 49 ------------ .../tests/test_get_submission_step.py | 50 ++++++++---- 10 files changed, 151 insertions(+), 181 deletions(-) delete mode 100644 src/openforms/custom_field_types/apps.py delete mode 100644 src/openforms/custom_field_types/family_members.py delete mode 100644 src/openforms/forms/custom_field_types.py diff --git a/src/openforms/custom_field_types/__init__.py b/src/openforms/custom_field_types/__init__.py index e69de29bb2..6485a7a552 100644 --- a/src/openforms/custom_field_types/__init__.py +++ b/src/openforms/custom_field_types/__init__.py @@ -0,0 +1,8 @@ +import warnings + +warnings.warn( + "The 'openforms.custom_field_types' is deprecated and will be removed when " + "the functionality to query 'arbitrary' data sources is finished. For more info, " + "see https://github.com/open-formulieren/open-forms/issues/2169.", + DeprecationWarning, +) diff --git a/src/openforms/custom_field_types/apps.py b/src/openforms/custom_field_types/apps.py deleted file mode 100644 index bd6db65fbb..0000000000 --- a/src/openforms/custom_field_types/apps.py +++ /dev/null @@ -1,8 +0,0 @@ -from django.apps import AppConfig - - -class CustomFieldTypesConfig(AppConfig): - name = "openforms.custom_field_types" - - def ready(self): - from . import family_members # noqa diff --git a/src/openforms/custom_field_types/family_members.py b/src/openforms/custom_field_types/family_members.py deleted file mode 100644 index 31c2b4f644..0000000000 --- a/src/openforms/custom_field_types/family_members.py +++ /dev/null @@ -1,58 +0,0 @@ -from typing import Any, Dict - -from openforms.authentication.constants import AuthAttribute -from openforms.formio.typing import Component -from openforms.forms.custom_field_types import register -from openforms.submissions.models import Submission - -from .constants import FamilyMembersDataAPIChoices -from .handlers.haal_centraal import get_np_children_haal_centraal -from .handlers.stuf_bg import get_np_children_stuf_bg -from .models import FamilyMembersTypeConfig - - -@register("npFamilyMembers") -def fill_out_family_members( - component: Component, submission: Submission -) -> Dict[str, Any]: - - # Check authentication details - if not submission.is_authenticated: - raise RuntimeError("No authenticated person!") - - if submission.auth_info.attribute != AuthAttribute.bsn: - raise RuntimeError("No BSN found in the authentication details.") - - bsn = submission.auth_info.value - - # Decide which API should be used to retrieve the children's data - config = FamilyMembersTypeConfig.get_solo() - handlers = { - FamilyMembersDataAPIChoices.haal_centraal: get_np_children_haal_centraal, - FamilyMembersDataAPIChoices.stuf_bg: get_np_children_stuf_bg, - } - - # Change the component configuration - component["type"] = "selectboxes" - component["fieldSet"] = False - component["inline"] = False - component["inputType"] = "checkbox" - - if not len(component["values"]) or component["values"][0] == { - "label": "", - "value": "", - }: - child_choices = handlers[config.data_api](bsn) - - component["values"] = [ - { - "label": label, - "value": value, - } - for value, label in child_choices - ] - - if "mask" in component: - del component["mask"] - - return component diff --git a/src/openforms/custom_field_types/tests/test_family_members.py b/src/openforms/custom_field_types/tests/test_family_members.py index 4d3cfa115a..4ee9110986 100644 --- a/src/openforms/custom_field_types/tests/test_family_members.py +++ b/src/openforms/custom_field_types/tests/test_family_members.py @@ -9,7 +9,7 @@ from openforms.authentication.constants import AuthAttribute from openforms.contrib.brp.models import BRPConfig -from openforms.forms.custom_field_types import handle_custom_types +from openforms.formio.service import get_dynamic_configuration from openforms.prefill.contrib.haalcentraal.tests.test_plugin import load_binary_mock from openforms.registrations.contrib.zgw_apis.tests.factories import ServiceFactory from openforms.submissions.tests.factories import SubmissionFactory @@ -24,14 +24,14 @@ class FamilyMembersCustomFieldTypeTest(TestCase): - @patch( - "openforms.custom_field_types.family_members.get_np_children_haal_centraal", - return_value=[("222333444", "Billy Doe"), ("333444555", "Jane Doe")], - ) - def test_get_values_for_custom_field(self, m): - configuration = { - "display": "form", - "components": [ + @patch("openforms.formio.components.custom.NPFamilyMembers._get_handler") + def test_get_values_for_custom_field(self, mock_get_handler): + mock_get_handler.return_value.return_value = [ + ("222333444", "Billy Doe"), + ("333444555", "Jane Doe"), + ] + submission = SubmissionFactory.from_components( + [ { "key": "npFamilyMembers", "type": "npFamilyMembers", @@ -39,32 +39,35 @@ def test_get_values_for_custom_field(self, m): "values": [{"label": "", "value": ""}], }, ], - } - submission = SubmissionFactory.create( auth_info__attribute=AuthAttribute.bsn, auth_info__value="111222333", - form__generate_minimal_setup=True, - form__formstep__form_definition__configuration=configuration, ) config = FamilyMembersTypeConfig.get_solo() config.data_api = FamilyMembersDataAPIChoices.haal_centraal config.save() + formio_wrapper = ( + submission.submissionstep_set.get().form_step.form_definition.configuration_wrapper + ) - rewritten_configuration = handle_custom_types(configuration, submission) + updated_config_wrapper = get_dynamic_configuration( + formio_wrapper, + request=None, + submission=submission, + ) - rewritten_component = rewritten_configuration["components"][0] + rewritten_component = updated_config_wrapper["npFamilyMembers"] self.assertEqual("selectboxes", rewritten_component["type"]) self.assertFalse(rewritten_component["fieldSet"]) self.assertFalse(rewritten_component["inline"]) self.assertEqual("checkbox", rewritten_component["inputType"]) self.assertEqual(2, len(rewritten_component["values"])) - self.assertDictEqual( - {"value": "222333444", "label": "Billy Doe"}, + self.assertEqual( rewritten_component["values"][0], + {"value": "222333444", "label": "Billy Doe"}, ) - self.assertDictEqual( - {"value": "333444555", "label": "Jane Doe"}, + self.assertEqual( rewritten_component["values"][1], + {"value": "333444555", "label": "Jane Doe"}, ) def test_get_children_haal_centraal(self): diff --git a/src/openforms/formio/components/custom.py b/src/openforms/formio/components/custom.py index ab660ecd6c..953b42e56c 100644 --- a/src/openforms/formio/components/custom.py +++ b/src/openforms/formio/components/custom.py @@ -1,11 +1,14 @@ import logging +from typing import Callable +from openforms.authentication.constants import AuthAttribute +from openforms.submissions.models import Submission from openforms.typing import DataMapping from openforms.utils.date import format_date_value from ..dynamic_config.date import FormioDateComponent, mutate as mutate_date from ..formatters.custom import DateFormatter, MapFormatter -from ..formatters.formio import TextFieldFormatter +from ..formatters.formio import DefaultFormatter, TextFieldFormatter from ..registry import BasePlugin, register from ..typing import Component from ..utils import conform_to_mask @@ -22,7 +25,7 @@ def normalizer(component: FormioDateComponent, value: str) -> str: return format_date_value(value) def mutate_config_dynamically( - self, component: FormioDateComponent, data: DataMapping + self, component: FormioDateComponent, submission: Submission, data: DataMapping ) -> None: """ Implement the behaviour for our custom date component options. @@ -58,3 +61,73 @@ def normalizer(component: Component, value: str) -> str: "Could not conform value '%s' to input mask '%s', returning original value." ) return value + + +@register("npFamilyMembers") +class NPFamilyMembers(BasePlugin): + # not actually relevant, as we transform the component into a different type + formatter = DefaultFormatter + + @staticmethod + def _get_handler() -> Callable[[str], list[tuple[str, str]]]: + # TODO: move these into a subpackage of openforms.formio + from openforms.custom_field_types.constants import FamilyMembersDataAPIChoices + from openforms.custom_field_types.handlers.haal_centraal import ( + get_np_children_haal_centraal, + ) + from openforms.custom_field_types.handlers.stuf_bg import ( + get_np_children_stuf_bg, + ) + from openforms.custom_field_types.models import FamilyMembersTypeConfig + + handlers = { + FamilyMembersDataAPIChoices.haal_centraal: get_np_children_haal_centraal, + FamilyMembersDataAPIChoices.stuf_bg: get_np_children_stuf_bg, + } + config = FamilyMembersTypeConfig.get_solo() + return handlers[config.data_api] + + @classmethod + def mutate_config_dynamically( + cls, component: Component, submission: Submission, data: DataMapping + ) -> None: + # Check authentication details/status before proceeding + if not submission.is_authenticated: + raise RuntimeError("No authenticated person!") + if submission.auth_info.attribute != AuthAttribute.bsn: + raise RuntimeError("No BSN found in the authentication details.") + + bsn = submission.auth_info.value + + component.update( + { + "type": "selectboxes", + "fieldSet": False, + "inline": False, + "inputType": "checkbox", + } + ) + + if "mask" in component: + del component["mask"] + + existing_values = component.get("values", []) + empty_option = { + "label": "", + "value": "", + } + if not existing_values or existing_values[0] == empty_option: + handler = cls._get_handler() + # make the API call + # TODO: this should eventually be replaced with logic rules/variables that + # retrieve data from an "arbitrary source", which will cause the data to + # become available in the ``data`` argument instead. + child_choices = handler(bsn) + + component["values"] = [ + { + "label": label, + "value": value, + } + for value, label in child_choices + ] diff --git a/src/openforms/formio/dynamic_config/__init__.py b/src/openforms/formio/dynamic_config/__init__.py index e405f01f84..e97604f826 100644 --- a/src/openforms/formio/dynamic_config/__init__.py +++ b/src/openforms/formio/dynamic_config/__init__.py @@ -7,6 +7,7 @@ from rest_framework.request import Request +from openforms.submissions.models import Submission from openforms.typing import DataMapping from ..datastructures import FormioConfigurationWrapper @@ -17,6 +18,7 @@ def rewrite_formio_components( configuration_wrapper: FormioConfigurationWrapper, + submission: Submission, data: Optional[DataMapping] = None, ) -> FormioConfigurationWrapper: """ @@ -33,7 +35,7 @@ def rewrite_formio_components( """ data = data or {} # normalize for component in configuration_wrapper: - register.update_config(component, data=data) + register.update_config(component, submission=submission, data=data) return configuration_wrapper diff --git a/src/openforms/formio/registry.py b/src/openforms/formio/registry.py index b0d8de444c..a2b84d6f06 100644 --- a/src/openforms/formio/registry.py +++ b/src/openforms/formio/registry.py @@ -21,7 +21,6 @@ from openforms.plugins.registry import BaseRegistry from openforms.typing import DataMapping -from .datastructures import FormioConfigurationWrapper from .typing import Component if TYPE_CHECKING: @@ -74,7 +73,7 @@ def verbose_name(self): return _("{type} component").format(type=self.identifier.capitalize()) def mutate_config_dynamically( - self, component: Component, data: DataMapping + self, component: Component, submission: "Submission", data: DataMapping ) -> None: # pragma: nocover ... @@ -109,7 +108,10 @@ def format(self, component: Component, value: Any, as_html=False): return formatter(component, value) def update_config( - self, component: Component, data: DataMapping | None = None + self, + component: Component, + submission: "Submission", + data: DataMapping | None = None, ) -> None: """ Mutate the component configuration in place. @@ -118,19 +120,19 @@ def update_config( for example) to work. """ # if there is no plugin registered for the component, return the input - if (component_type := component["type"]) not in register: + if (component_type := component["type"]) not in self: return # invoke plugin if exists plugin = self[component_type] - plugin.mutate_config_dynamically(component, data) + plugin.mutate_config_dynamically(component, submission, data) def update_config_for_request(self, component: Component, request: Request) -> None: """ Mutate the component in place for the given request context. """ # if there is no plugin registered for the component, return the input - if (component_type := component["type"]) not in register: + if (component_type := component["type"]) not in self: return # invoke plugin if exists @@ -140,21 +142,6 @@ def update_config_for_request(self, component: Component, request: Request) -> N rewriter(component, request) - def handle_custom_types( - self, - configuration: FormioConfigurationWrapper, - submission: "Submission", - ): - """ - Process custom backend-only formio types. - - Formio types can be transformed in the context of a given - :class:`openforms.submissions.models.Submission` and ultimately manifest as - modified or standard Formio types, essentially performing some sort of "rewrite" - of the Formio configuration object. - """ - raise NotImplementedError() - # Sentinel to provide the default registry. You can easily instantiate another # :class:`Registry` object to use as dependency injection in tests. diff --git a/src/openforms/formio/service.py b/src/openforms/formio/service.py index 28d91bf47e..faf101c2ae 100644 --- a/src/openforms/formio/service.py +++ b/src/openforms/formio/service.py @@ -13,7 +13,6 @@ import elasticapm from rest_framework.request import Request -from openforms.forms.custom_field_types import handle_custom_types from openforms.prefill import inject_prefill from openforms.submissions.models import Submission from openforms.typing import DataMapping @@ -62,12 +61,7 @@ def get_dynamic_configuration( The configuration is modified in the context of the provided :arg:`submission`. """ - # TODO: see if we can make the config wrapper smart enough to deal with this - config_wrapper.configuration = handle_custom_types( - config_wrapper.configuration, submission=submission - ) - - rewrite_formio_components(config_wrapper, data=data) + rewrite_formio_components(config_wrapper, submission=submission, data=data) # prefill is still 'special' even though it uses variables, as we specifically # set the `defaultValue` key to the resulting variable. diff --git a/src/openforms/forms/custom_field_types.py b/src/openforms/forms/custom_field_types.py deleted file mode 100644 index 37899deb47..0000000000 --- a/src/openforms/forms/custom_field_types.py +++ /dev/null @@ -1,49 +0,0 @@ -from typing import Any, Dict - -import elasticapm - -from openforms.submissions.models import Submission - -__all__ = ["register", "unregister", "handle_custom_types"] - -REGISTRY = {} - - -def register(custom_type: str): - def decorator(handler: callable): - if custom_type in REGISTRY: - raise ValueError(f"Custom type {custom_type} is already registered.") - - REGISTRY[custom_type] = handler - - return decorator - - -def unregister(custom_type: str): - if custom_type in REGISTRY: - del REGISTRY[custom_type] - - -@elasticapm.capture_span(span_type="app.formio") -def handle_custom_types( - configuration: Dict[str, Any], - submission: Submission, -) -> Dict[str, Any]: - - rewritten_components = [] - - for component in configuration["components"]: - type_key = component["type"] - - # no handler -> leave untouched - if type_key not in REGISTRY: - rewritten_components.append(component) - continue - - # if there is a handler, invoke it - handler = REGISTRY[type_key] - rewritten_components.append(handler(component, submission)) - - return { - "components": rewritten_components, - } diff --git a/src/openforms/submissions/tests/test_get_submission_step.py b/src/openforms/submissions/tests/test_get_submission_step.py index 201fb3fa2d..a12657405c 100644 --- a/src/openforms/submissions/tests/test_get_submission_step.py +++ b/src/openforms/submissions/tests/test_get_submission_step.py @@ -9,6 +9,7 @@ aware step definition. """ import uuid +from unittest.mock import patch from django.test import tag @@ -17,7 +18,9 @@ from rest_framework.test import APITestCase from openforms.accounts.tests.factories import StaffUserFactory, SuperUserFactory -from openforms.forms.custom_field_types import register, unregister +from openforms.formio.components.vanilla import SelectBoxes, TextField +from openforms.formio.formatters.formio import TextFieldFormatter +from openforms.formio.registry import BasePlugin, ComponentRegistry from openforms.forms.tests.factories import ( FormFactory, FormLogicFactory, @@ -112,15 +115,21 @@ def test_static_form_definition(self): self.assertEqual(response.json(), expected) def test_dynamic_form_definition(self): + register = ComponentRegistry() + register("selectboxes")(SelectBoxes) + @register("textfield") - def custom_handler(component: dict, request, submission): - component["label"] = "Rewritten label" - return component + class TextField(BasePlugin): + formatter = TextFieldFormatter + + @staticmethod + def mutate_config_dynamically(component, submission, data): + component["label"] = "Rewritten label" - self.addCleanup(lambda: unregister("textfield")) self._add_submission_to_session(self.submission) - response = self.client.get(self.step_url) + with patch("openforms.formio.dynamic_config.register", new=register): + response = self.client.get(self.step_url) self.assertEqual(response.status_code, status.HTTP_200_OK) expected = { @@ -443,18 +452,26 @@ def test_dynamic_date_component_config_based_on_variables(self): self.assertEqual(date2["datePicker"]["maxDate"], "2022-12-31T00:00:00+01:00") def test_custom_components_and_form_logic(self): - # set up custom field type for test only - self.addCleanup(lambda: unregister("testCustomType")) + register = ComponentRegistry() + register("textfield")(TextField) @register("testCustomType") - def custom_type(component, request, submission): - return { - "type": "textfield", - "key": component["key"], - "defaultValue": "testCustomType", - "hidden": False, - } + class CustomType(BasePlugin): + formatter = TextFieldFormatter + + @staticmethod + def mutate_config_dynamically(component, submission, data): + key = component["key"] + component.clear() + component.update( + { + "type": "textfield", + "key": key, + "defaultValue": "testCustomType", + "hidden": False, + } + ) form = FormFactory.create( generate_minimal_setup=True, @@ -504,7 +521,8 @@ def custom_type(component, request, submission): ) self._add_submission_to_session(submission) - response = self.client.get(endpoint) + with patch("openforms.formio.dynamic_config.register", new=register): + response = self.client.get(endpoint) self.assertEqual(response.status_code, status.HTTP_200_OK) components = response.data["form_step"]["configuration"]["components"] From 6b9dea1c11eb94b40bf87be61eb5e9e08bba35a1 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 16:43:21 +0100 Subject: [PATCH 11/16] :card_file_box: [#1068] -- prepare to move model to other app --- .../0002_alter_familymemberstypeconfig_table.py | 17 +++++++++++++++++ src/openforms/custom_field_types/models.py | 1 + 2 files changed, 18 insertions(+) create mode 100644 src/openforms/custom_field_types/migrations/0002_alter_familymemberstypeconfig_table.py diff --git a/src/openforms/custom_field_types/migrations/0002_alter_familymemberstypeconfig_table.py b/src/openforms/custom_field_types/migrations/0002_alter_familymemberstypeconfig_table.py new file mode 100644 index 0000000000..ee531f3413 --- /dev/null +++ b/src/openforms/custom_field_types/migrations/0002_alter_familymemberstypeconfig_table.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.16 on 2022-11-24 15:42 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("custom_field_types", "0001_initial"), + ] + + operations = [ + migrations.AlterModelTable( + name="familymemberstypeconfig", + table="custom_field_types_familymemberstypeconfig", + ), + ] diff --git a/src/openforms/custom_field_types/models.py b/src/openforms/custom_field_types/models.py index 0e043aad4f..c68c524727 100644 --- a/src/openforms/custom_field_types/models.py +++ b/src/openforms/custom_field_types/models.py @@ -16,3 +16,4 @@ class FamilyMembersTypeConfig(SingletonModel): class Meta: verbose_name = _("Family members type configuration") + db_table = "custom_field_types_familymemberstypeconfig" From 45b5a526d90ecf64b6e39283a20a7e7d73c7a1b0 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 16:55:47 +0100 Subject: [PATCH 12/16] :truck: [#1068] -- move npFamilyMembers code into openforms.formio package --- src/openforms/conf/base.py | 3 +- .../0003_delete_familymemberstypeconfig.py | 21 ++++++++ .../fixtures/default_admin_index.json | 2 +- src/openforms/formio/components/custom.py | 14 ++---- .../components/np_family_members}/__init__.py | 0 .../components/np_family_members}/admin.py | 0 .../np_family_members}/constants.py | 0 .../np_family_members}/haal_centraal.py | 0 .../migrations/0001_initial.py | 50 +++++++++++++++++++ ...002_alter_familymemberstypeconfig_table.py | 17 +++++++ .../np_family_members/migrations}/__init__.py | 0 .../components/np_family_members}/models.py | 1 - .../components/np_family_members}/stuf_bg.py | 0 .../np_family_members/tests/__init__.py | 0 .../tests/responses/op_2_children.json | 0 .../tests/responses/stuf_bg_2_children.xml | 0 .../tests/test_family_members.py | 4 +- 17 files changed, 97 insertions(+), 15 deletions(-) create mode 100644 src/openforms/custom_field_types/migrations/0003_delete_familymemberstypeconfig.py rename src/openforms/{custom_field_types/handlers => formio/components/np_family_members}/__init__.py (100%) rename src/openforms/{custom_field_types => formio/components/np_family_members}/admin.py (100%) rename src/openforms/{custom_field_types => formio/components/np_family_members}/constants.py (100%) rename src/openforms/{custom_field_types/handlers => formio/components/np_family_members}/haal_centraal.py (100%) create mode 100644 src/openforms/formio/components/np_family_members/migrations/0001_initial.py create mode 100644 src/openforms/formio/components/np_family_members/migrations/0002_alter_familymemberstypeconfig_table.py rename src/openforms/{custom_field_types/tests => formio/components/np_family_members/migrations}/__init__.py (100%) rename src/openforms/{custom_field_types => formio/components/np_family_members}/models.py (89%) rename src/openforms/{custom_field_types/handlers => formio/components/np_family_members}/stuf_bg.py (100%) create mode 100644 src/openforms/formio/components/np_family_members/tests/__init__.py rename src/openforms/{custom_field_types => formio/components/np_family_members}/tests/responses/op_2_children.json (100%) rename src/openforms/{custom_field_types => formio/components/np_family_members}/tests/responses/stuf_bg_2_children.xml (100%) rename src/openforms/{custom_field_types => formio/components/np_family_members}/tests/test_family_members.py (97%) diff --git a/src/openforms/conf/base.py b/src/openforms/conf/base.py index aca410fe9a..1591cc2a39 100644 --- a/src/openforms/conf/base.py +++ b/src/openforms/conf/base.py @@ -193,6 +193,7 @@ "openforms.config", "openforms.emails", "openforms.formio", + "openforms.formio.components.np_family_members", "openforms.formio.rendering", "openforms.forms", "openforms.multidomain", @@ -236,7 +237,7 @@ "openforms.utils", "openforms.upgrades", "openforms.plugins", - "openforms.custom_field_types", + "openforms.custom_field_types", # TODO: remove completely after 2.1 is released # Apps registering static variables "openforms.variables.static_variables.apps.StaticVariables", "openforms.authentication.static_variables.apps.AuthStaticVariables", diff --git a/src/openforms/custom_field_types/migrations/0003_delete_familymemberstypeconfig.py b/src/openforms/custom_field_types/migrations/0003_delete_familymemberstypeconfig.py new file mode 100644 index 0000000000..befd11bd26 --- /dev/null +++ b/src/openforms/custom_field_types/migrations/0003_delete_familymemberstypeconfig.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.16 on 2022-11-24 15:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("custom_field_types", "0002_alter_familymemberstypeconfig_table"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.DeleteModel( + name="FamilyMembersTypeConfig", + ), + ], + database_operations=[], + ), + ] diff --git a/src/openforms/fixtures/default_admin_index.json b/src/openforms/fixtures/default_admin_index.json index bc70b653d4..65ec56b149 100644 --- a/src/openforms/fixtures/default_admin_index.json +++ b/src/openforms/fixtures/default_admin_index.json @@ -155,7 +155,7 @@ "cookiegroup" ], [ - "custom_field_types", + "np_family_members", "familymemberstypeconfig" ], [ diff --git a/src/openforms/formio/components/custom.py b/src/openforms/formio/components/custom.py index 953b42e56c..bb4c261f80 100644 --- a/src/openforms/formio/components/custom.py +++ b/src/openforms/formio/components/custom.py @@ -12,6 +12,10 @@ from ..registry import BasePlugin, register from ..typing import Component from ..utils import conform_to_mask +from .np_family_members.constants import FamilyMembersDataAPIChoices +from .np_family_members.haal_centraal import get_np_children_haal_centraal +from .np_family_members.models import FamilyMembersTypeConfig +from .np_family_members.stuf_bg import get_np_children_stuf_bg logger = logging.getLogger(__name__) @@ -70,16 +74,6 @@ class NPFamilyMembers(BasePlugin): @staticmethod def _get_handler() -> Callable[[str], list[tuple[str, str]]]: - # TODO: move these into a subpackage of openforms.formio - from openforms.custom_field_types.constants import FamilyMembersDataAPIChoices - from openforms.custom_field_types.handlers.haal_centraal import ( - get_np_children_haal_centraal, - ) - from openforms.custom_field_types.handlers.stuf_bg import ( - get_np_children_stuf_bg, - ) - from openforms.custom_field_types.models import FamilyMembersTypeConfig - handlers = { FamilyMembersDataAPIChoices.haal_centraal: get_np_children_haal_centraal, FamilyMembersDataAPIChoices.stuf_bg: get_np_children_stuf_bg, diff --git a/src/openforms/custom_field_types/handlers/__init__.py b/src/openforms/formio/components/np_family_members/__init__.py similarity index 100% rename from src/openforms/custom_field_types/handlers/__init__.py rename to src/openforms/formio/components/np_family_members/__init__.py diff --git a/src/openforms/custom_field_types/admin.py b/src/openforms/formio/components/np_family_members/admin.py similarity index 100% rename from src/openforms/custom_field_types/admin.py rename to src/openforms/formio/components/np_family_members/admin.py diff --git a/src/openforms/custom_field_types/constants.py b/src/openforms/formio/components/np_family_members/constants.py similarity index 100% rename from src/openforms/custom_field_types/constants.py rename to src/openforms/formio/components/np_family_members/constants.py diff --git a/src/openforms/custom_field_types/handlers/haal_centraal.py b/src/openforms/formio/components/np_family_members/haal_centraal.py similarity index 100% rename from src/openforms/custom_field_types/handlers/haal_centraal.py rename to src/openforms/formio/components/np_family_members/haal_centraal.py diff --git a/src/openforms/formio/components/np_family_members/migrations/0001_initial.py b/src/openforms/formio/components/np_family_members/migrations/0001_initial.py new file mode 100644 index 0000000000..63e6fdb65a --- /dev/null +++ b/src/openforms/formio/components/np_family_members/migrations/0001_initial.py @@ -0,0 +1,50 @@ +# Generated by Django 3.2.16 on 2022-11-24 15:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("custom_field_types", "0002_alter_familymemberstypeconfig_table"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.CreateModel( + name="FamilyMembersTypeConfig", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "data_api", + models.CharField( + choices=[ + ("haal_centraal", "Haal Centraal"), + ("stuf_bg", "StufBg"), + ], + help_text="Which API to use to retrieve the data of the family members.", + max_length=100, + verbose_name="data api", + ), + ), + ], + options={ + "verbose_name": "Family members type configuration", + "db_table": "custom_field_types_familymemberstypeconfig", + }, + ), + ], + database_operations=[], + ), + ] diff --git a/src/openforms/formio/components/np_family_members/migrations/0002_alter_familymemberstypeconfig_table.py b/src/openforms/formio/components/np_family_members/migrations/0002_alter_familymemberstypeconfig_table.py new file mode 100644 index 0000000000..6e1ec44101 --- /dev/null +++ b/src/openforms/formio/components/np_family_members/migrations/0002_alter_familymemberstypeconfig_table.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.16 on 2022-11-24 15:56 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("np_family_members", "0001_initial"), + ] + + operations = [ + migrations.AlterModelTable( + name="familymemberstypeconfig", + table=None, + ), + ] diff --git a/src/openforms/custom_field_types/tests/__init__.py b/src/openforms/formio/components/np_family_members/migrations/__init__.py similarity index 100% rename from src/openforms/custom_field_types/tests/__init__.py rename to src/openforms/formio/components/np_family_members/migrations/__init__.py diff --git a/src/openforms/custom_field_types/models.py b/src/openforms/formio/components/np_family_members/models.py similarity index 89% rename from src/openforms/custom_field_types/models.py rename to src/openforms/formio/components/np_family_members/models.py index c68c524727..0e043aad4f 100644 --- a/src/openforms/custom_field_types/models.py +++ b/src/openforms/formio/components/np_family_members/models.py @@ -16,4 +16,3 @@ class FamilyMembersTypeConfig(SingletonModel): class Meta: verbose_name = _("Family members type configuration") - db_table = "custom_field_types_familymemberstypeconfig" diff --git a/src/openforms/custom_field_types/handlers/stuf_bg.py b/src/openforms/formio/components/np_family_members/stuf_bg.py similarity index 100% rename from src/openforms/custom_field_types/handlers/stuf_bg.py rename to src/openforms/formio/components/np_family_members/stuf_bg.py diff --git a/src/openforms/formio/components/np_family_members/tests/__init__.py b/src/openforms/formio/components/np_family_members/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/openforms/custom_field_types/tests/responses/op_2_children.json b/src/openforms/formio/components/np_family_members/tests/responses/op_2_children.json similarity index 100% rename from src/openforms/custom_field_types/tests/responses/op_2_children.json rename to src/openforms/formio/components/np_family_members/tests/responses/op_2_children.json diff --git a/src/openforms/custom_field_types/tests/responses/stuf_bg_2_children.xml b/src/openforms/formio/components/np_family_members/tests/responses/stuf_bg_2_children.xml similarity index 100% rename from src/openforms/custom_field_types/tests/responses/stuf_bg_2_children.xml rename to src/openforms/formio/components/np_family_members/tests/responses/stuf_bg_2_children.xml diff --git a/src/openforms/custom_field_types/tests/test_family_members.py b/src/openforms/formio/components/np_family_members/tests/test_family_members.py similarity index 97% rename from src/openforms/custom_field_types/tests/test_family_members.py rename to src/openforms/formio/components/np_family_members/tests/test_family_members.py index 4ee9110986..868e279798 100644 --- a/src/openforms/custom_field_types/tests/test_family_members.py +++ b/src/openforms/formio/components/np_family_members/tests/test_family_members.py @@ -18,8 +18,8 @@ from stuf.tests.factories import StufServiceFactory from ..constants import FamilyMembersDataAPIChoices -from ..handlers.haal_centraal import get_np_children_haal_centraal -from ..handlers.stuf_bg import get_np_children_stuf_bg +from ..haal_centraal import get_np_children_haal_centraal +from ..stuf_bg import get_np_children_stuf_bg from ..models import FamilyMembersTypeConfig From 5c6532cc6d2c8a0b6b3c168be81e0850d3c2c6f7 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 17:14:48 +0100 Subject: [PATCH 13/16] :pencil: [#1068] -- fix outdated docstring --- src/openforms/formio/dynamic_config/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/openforms/formio/dynamic_config/__init__.py b/src/openforms/formio/dynamic_config/__init__.py index e97604f826..71ea9ab702 100644 --- a/src/openforms/formio/dynamic_config/__init__.py +++ b/src/openforms/formio/dynamic_config/__init__.py @@ -27,11 +27,11 @@ def rewrite_formio_components( :arg configuration_wrapper: Container object holding the Formio form configuration to be updated (if applicable). The rewriting loops over all components one-by-one and applies the changes. + :arg submission: The submission instance for which we are rewriting the + configurations. This allows you to inspect state and use convenience methods + that may not be available in the variables data. :arg data: key-value mapping of variable name to variable value. If a submission context is available, the variables of the submission are included here. - :arg request: The HTTP request received by the API view when rewriting is done as - part of an HTTP request-response cycle. This is ``None`` otherwise (e.g. in - background tasks). """ data = data or {} # normalize for component in configuration_wrapper: From fdf077953d1de9b7bad6afaecefd6184dcd023a0 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 17:55:36 +0100 Subject: [PATCH 14/16] :truck: [#1068] -- reorganize backend documentation The navtree was confusing and too nested, this structures the docs more along the core/modules principles and keeps the topics on a single page. --- docs/developers/backend/core/index.rst | 51 ++++-------- .../backend/core/submission-renderer.rst | 6 +- docs/developers/backend/core/submissions.rst | 6 +- .../developers/backend/core/testing-tools.rst | 14 +++- docs/developers/backend/core/variables.rst | 11 +-- docs/developers/backend/index.rst | 78 ++++++++++++++++--- .../tests/test_family_members.py | 2 +- 7 files changed, 111 insertions(+), 57 deletions(-) diff --git a/docs/developers/backend/core/index.rst b/docs/developers/backend/core/index.rst index cf54895abe..ed057de8eb 100644 --- a/docs/developers/backend/core/index.rst +++ b/docs/developers/backend/core/index.rst @@ -1,44 +1,27 @@ .. _developers_backend_core_index: ================== -Core functionality +Core: introduction ================== -Reference documentation for the Open Forms core. +As stated in the :ref:`developers_architecture`: -General principles -================== - -**Keep Django apps contained** - -Django apps should focus on a single responsibility with minimal dependencies on the -"outside world". - -**Explicitly expose public API** - -Usually we only consider the ``service.py`` module of a Django app to be public API. -Please think twice before introducing breaking changes in those modules. - -**Refrain from importing private API** - -Importing things from modules in core functionality is generally frowned upon ( -``service.py`` is the exception here). Modules should be able to freely alter their -implementation details, including their data model! - -**Document useful, generic functionality** + Core functionality is considered functionality that does not or very loosely tie in + to particular modules. It is functionality that has meaning on its own without + dependencies, but is enriched by modules. -Documentation makes it easier to find out what exists and avoid re-implementing the -same thing twice. +The core really implements the "engine" of Open Forms and hides all the implementation +details. It should be fairly stable, but also continually allow for new feature +additions, which is a challenging task! +The following Django apps are considered core functionality: -.. toctree:: - :caption: Contents - :maxdepth: 2 +* :mod:`openforms.accounts`: (staff) user account management +* :mod:`openforms.config`: application-wide configuration and defaults +* :mod:`openforms.forms`: designing and building of forms +* :mod:`openforms.formio`: integration with the `form.io`_ frontend library +* :mod:`openforms.submissions`: persisting and handling of submitted form data +* :mod:`openforms.variables`: persisting (intermediate) data into variables for further + operations - submissions - submission-renderer - utils - tokens - testing-tools - variables - templating +.. _form.io: https://www.form.io/ diff --git a/docs/developers/backend/core/submission-renderer.rst b/docs/developers/backend/core/submission-renderer.rst index 818f42d4c4..052c152704 100644 --- a/docs/developers/backend/core/submission-renderer.rst +++ b/docs/developers/backend/core/submission-renderer.rst @@ -2,9 +2,9 @@ .. currentmodule:: openforms.submissions.rendering -==================== -Submission rendering -==================== +========================== +Core: submission rendering +========================== Submission rendering is the concept of outputting the submitted data for a given form in a particular format. A couple of examples: diff --git a/docs/developers/backend/core/submissions.rst b/docs/developers/backend/core/submissions.rst index 600e22e431..e4a7563f8e 100644 --- a/docs/developers/backend/core/submissions.rst +++ b/docs/developers/backend/core/submissions.rst @@ -1,8 +1,8 @@ .. _developers_backend_core_submissions: -=========== -Submissions -=========== +=========================== +Core: submission processing +=========================== Once a form is submitted by the user, a number of actions take place to process this submission. The complete process is shown below. diff --git a/docs/developers/backend/core/testing-tools.rst b/docs/developers/backend/core/testing-tools.rst index 6aae896d6e..e30b6b968c 100644 --- a/docs/developers/backend/core/testing-tools.rst +++ b/docs/developers/backend/core/testing-tools.rst @@ -1,7 +1,11 @@ .. _developers_backend_core_testing_tools: -Testing tools -============= +============ +Test helpers +============ + +HTML assertions +=============== .. automodule:: openforms.utils.tests.html_assert :members: @@ -9,9 +13,15 @@ Testing tools .. automodule:: openforms.utils.tests.webtest_base :members: +Migrations +========== + .. automodule:: openforms.utils.tests.test_migrations :members: +Formio assertions +================= + .. automodule:: openforms.formio.tests.assertions :members: :undoc-members: diff --git a/docs/developers/backend/core/variables.rst b/docs/developers/backend/core/variables.rst index 4026ec7f50..3884d4c1b7 100644 --- a/docs/developers/backend/core/variables.rst +++ b/docs/developers/backend/core/variables.rst @@ -1,12 +1,13 @@ .. _developers_backend_core_variables: -========= -Variables -========= +=============== +Core: variables +=============== There are two models for variables, :class:`openforms.forms.models.FormVariable` and -:class:`openforms.submissions.models.SubmissionValueVariable`. ``FormVariable`` are related to -a form while ``SubmissionValueVariable`` are related to a submission. +:class:`openforms.submissions.models.SubmissionValueVariable`. ``FormVariable`` objects +are related to a form while ``SubmissionValueVariable`` objects are related to a +submission (and a form variable). Form Variables ============== diff --git a/docs/developers/backend/index.rst b/docs/developers/backend/index.rst index f62ecafc48..8f7cd34f9e 100644 --- a/docs/developers/backend/index.rst +++ b/docs/developers/backend/index.rst @@ -10,15 +10,75 @@ Django has a concept of "apps" within a project which logically contain some functionality. The codebase layout reflects this - but the documentation is structured following the outlined :ref:`developers_architecture`. +General principles +================== + +On top of all this, we apply some general principles to keep/make the codebase +maintainable. + +**Keep Django apps contained** + +Django apps should focus on a single responsibility with minimal dependencies on the +"outside world". + +**Explicitly expose public API** + +Usually we only consider the ``service.py`` module of a Django app to be public API. +Please think twice before introducing breaking changes in those modules. + +**Refrain from importing private API** + +Importing things from modules in core functionality is generally frowned upon ( +``service.py`` is the exception here). Modules should be able to freely alter their +implementation details, including their data model! + +**Document useful, generic functionality** + +Documentation makes it easier to find out what exists and avoid re-implementing the +same thing twice. + +Core +==== + .. toctree:: - :caption: Contents - :maxdepth: 2 - - core/index - modules/index - dev-rendering - profiling - upgrade-checks - file-uploads + :maxdepth: 1 + + core/index + core/submissions + core/submission-renderer + core/variables + file-uploads + +Modules +======= + +.. toctree:: + :maxdepth: 1 + + modules/index + modules/dmn + +General purpose functionality +============================= + +.. toctree:: + :maxdepth: 1 + + core/utils + core/tokens + core/testing-tools + core/templating + + upgrade-checks + +Development and debug tooling +============================= + +.. toctree:: + :maxdepth: 1 + + dev-rendering + profiling + file-uploads .. _Django: https://www.djangoproject.com/ diff --git a/src/openforms/formio/components/np_family_members/tests/test_family_members.py b/src/openforms/formio/components/np_family_members/tests/test_family_members.py index 868e279798..9e8f20a6c6 100644 --- a/src/openforms/formio/components/np_family_members/tests/test_family_members.py +++ b/src/openforms/formio/components/np_family_members/tests/test_family_members.py @@ -19,8 +19,8 @@ from ..constants import FamilyMembersDataAPIChoices from ..haal_centraal import get_np_children_haal_centraal -from ..stuf_bg import get_np_children_stuf_bg from ..models import FamilyMembersTypeConfig +from ..stuf_bg import get_np_children_stuf_bg class FamilyMembersCustomFieldTypeTest(TestCase): From 7f18c52481d8a21c1d253b8bd5b61a796711c157 Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 18:34:29 +0100 Subject: [PATCH 15/16] :pencil: [#1068] -- document formio service and utilities --- docs/developers/backend/core/formio.rst | 134 ++++++++++++++++++ docs/developers/backend/index.rst | 1 + docs/developers/backend/modules/index.rst | 12 +- .../formio/dynamic_config/__init__.py | 2 - src/openforms/formio/service.py | 6 +- 5 files changed, 143 insertions(+), 12 deletions(-) create mode 100644 docs/developers/backend/core/formio.rst diff --git a/docs/developers/backend/core/formio.rst b/docs/developers/backend/core/formio.rst new file mode 100644 index 0000000000..d71fc23082 --- /dev/null +++ b/docs/developers/backend/core/formio.rst @@ -0,0 +1,134 @@ +.. _developers_backend_core_formio: + +========================= +Core: form.io integration +========================= + +Open Forms uses `form.io`_ under the hood to build and render forms, and then adds its +own layers on top of that, such as: + +* implementing multi-step forms where every step is a form.io definition +* evaluating backend logic using data from earlier steps +* dynamically adapting form.io definitions as needed + +This means that we process the form.io datastructures in the backend, using Python. All +the code for this is organized in the ``openforms.formio`` package. + +.. versionchanged:: 2.1.0 + + ``openforms.custom_field_types`` was refactored into the ``openforms.formio`` package, + and all of the separate registries (formatters, normalizers...) were merged into a + single compoment registry. + +Supported features +================== + +.. currentmodule:: openforms.formio.service + +Formatting values for display +----------------------------- + +Value formatting is done for displaying form submission data summaries, rendering +confirmation PDFs and emails... It is aware if it's in a HTML context or not. It is +heavily used in the :ref:`renderers `. + +Whenever a component plugin is registered, the +:attr:`openforms.formio.registry.BasePlugin.formatter` class attribute **must** be +specified. + +.. autofunction:: format_value + :noindex: + + +Normalizing input data +---------------------- + +Data for a component can be sourced from external systems that employ different +formatting rules compared to what form.io expects. Normalizing this data helps to be +able to make proper comparisons at different stages in the submission life-cycle. + +You can opt-in to this by configuring :attr:`openforms.formio.registry.BasePlugin.normalizer`. + +.. autofunction:: normalize_value_for_component + :noindex: + + +Dynamically modifying component configuration +--------------------------------------------- + +Certain component types require on-the-fly configuration rewriting, such as applying +global configuration options that may change independently from when the form is +actually being designed. + +Dynamic rewriting is enabled by implementing +:meth:`openforms.formio.registry.BasePlugin.mutate_config_dynamically`. It receives the +current :class:`openforms.submissions.models.Submission` instance and a mapping of all +the variable names and values at the time. + +.. autofunction:: get_dynamic_configuration + :noindex: + +For an example of a custom field type, see :class:`openforms.formio.components.custom.Date`. + +Finally, the resulting resolved component definitions are evaluated with the template +engine where variable values are evaluated for compoment labels, descriptions... and +configuration based on the HTTP request is performed (see +:func:`openforms.formmio.service.rewrite_formio_components_for_request`). + +Reference +========= + +Public API - ``openforms.formio.service`` +----------------------------------------- + +.. automodule:: openforms.formio.service + :members: + +.. autoclass:: openforms.formio.registry.BasePlugin + :members: + :exclude-members: verbose_name + +Extending +--------- + +Using our :ref:`usual extension pattern ` you can register your +own types. + +Extensions should inherit from :class:`openforms.formio.registry.BasePlugin` or +implement the same protocol(s) and be registered with their form.io type: + +.. code-block:: python + + from openforms.formio.formatters.formio import DefaultFormatter + from openforms.formio.registry import BasePlugin + + @register("myCustomType") + class MyComponent(BasePlugin): + formatter = DefaultFormatter + + +You can find some examples in ``openforms.formio.components.custom``. + + +Private API +----------- + +Module: ``openforms.formio.dynamic_config`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: openforms.formio.dynamic_config + :members: + +Module: ``openforms.formio.formatters`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: openforms.formio.formatters + :members: + +Module: ``openforms.formio.rendering`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. automodule:: openforms.formio.rendering + :members: + +.. _form.io: https://www.form.io/ diff --git a/docs/developers/backend/index.rst b/docs/developers/backend/index.rst index 8f7cd34f9e..6860596734 100644 --- a/docs/developers/backend/index.rst +++ b/docs/developers/backend/index.rst @@ -44,6 +44,7 @@ Core :maxdepth: 1 core/index + core/formio core/submissions core/submission-renderer core/variables diff --git a/docs/developers/backend/modules/index.rst b/docs/developers/backend/modules/index.rst index 940d168fc5..01d3184b70 100644 --- a/docs/developers/backend/modules/index.rst +++ b/docs/developers/backend/modules/index.rst @@ -1,8 +1,8 @@ .. _developers_backend_modules_index: -======= -Modules -======= +===================== +Modules: introduction +===================== Open Forms groups functionalities in *modules*. Within a module there are often one or more plugins available for a vendor-specific implementation of the high-level @@ -10,9 +10,3 @@ functionality. Modules can usually be :ref:`extended ` wit plugins. .. todo:: This documentation is incomplete - -.. toctree:: - :caption: Contents - :maxdepth: 2 - - dmn diff --git a/src/openforms/formio/dynamic_config/__init__.py b/src/openforms/formio/dynamic_config/__init__.py index 71ea9ab702..dfb0afbda1 100644 --- a/src/openforms/formio/dynamic_config/__init__.py +++ b/src/openforms/formio/dynamic_config/__init__.py @@ -1,7 +1,5 @@ """ Implement component-specific dynamic configuration based on (submission) state. - -This will eventually replace ``openforms.forms.custom_field_types``. """ from typing import Optional diff --git a/src/openforms/formio/service.py b/src/openforms/formio/service.py index faf101c2ae..956eb62a61 100644 --- a/src/openforms/formio/service.py +++ b/src/openforms/formio/service.py @@ -38,6 +38,9 @@ def format_value(component: Component, value: Any, as_html: bool = False): + """ + Format a submitted value in a way that is most appropriate for the component type. + """ return register.format(component, value, as_html=as_html) @@ -59,7 +62,8 @@ def get_dynamic_configuration( """ Given a static Formio configuration, apply the hooks to dynamically transform this. - The configuration is modified in the context of the provided :arg:`submission`. + The configuration is modified in the context of the provided ``submission`` + parameter. """ rewrite_formio_components(config_wrapper, submission=submission, data=data) From 5b2629fff7655bcf4e75c085ccc4aac334185d4c Mon Sep 17 00:00:00 2001 From: Sergei Maertens Date: Thu, 24 Nov 2022 21:30:55 +0100 Subject: [PATCH 16/16] :white_check_mark: [#1068] -- improving test coverage --- setup.cfg | 22 +--------- .../tests/test_family_members.py | 11 +++-- src/openforms/formio/datastructures.py | 11 ----- .../tests/test_file_component.py | 41 +++++++++++++++++++ src/openforms/formio/formatters/base.py | 2 +- src/openforms/formio/formatters/service.py | 15 ------- src/openforms/formio/registry.py | 8 ++-- .../formio/tests/test_normalization.py | 7 ++++ .../submissions/models/submission.py | 2 +- 9 files changed, 61 insertions(+), 58 deletions(-) create mode 100644 src/openforms/formio/dynamic_config/tests/test_file_component.py delete mode 100644 src/openforms/formio/formatters/service.py diff --git a/setup.cfg b/setup.cfg index a1e93f13e5..16f57e646c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,9 @@ source = src omit = manage.py */tests/test_* + */tests/factories.py */migrations/* + src/openforms/formio/formatters/tests/mixins.py src/openforms/submissions/management/commands/test_submission_completion.py src/openforms/submissions/management/commands/render_confirmation_pdf.py src/openforms/formio/management/commands/formio_makemessages.py @@ -51,23 +53,3 @@ omit = [coverage:report] skip_covered = True -omit = - manage.py - */tests/test_* - */tests/factories.py - */migrations/* - src/openforms/submissions/management/commands/test_submission_completion.py - src/openforms/submissions/management/commands/render_confirmation_pdf.py - src/openforms/formio/management/commands/formio_makemessages.py - src/openforms/appointments/management/commands/appointment.py - src/openforms/registrations/management/commands/register_submission.py - src/openforms/registrations/contrib/microsoft_graph/management/commands/msgraph_list_files.py - src/openforms/prefill/management/commands/generate_prefill_from_spec.py - src/openforms/utils/management/commands/check_admin_index.py - src/openforms/registrations/contrib/stuf_zds/management/commands/stuf_zds_test_stp.py - src/openforms/plugins/management/commands/disable_demo_plugins.py - src/openforms/payments/management/commands/checkpaymentemaildupes.py - src/openforms/registrations/contrib/email/views.py - src/openforms/upgrades/tests/files/check_fail.py-tpl - src/openforms/upgrades/tests/files/check_pass.py-tpl - src/openforms/upgrades/tests/files/check_error.py-tpl diff --git a/src/openforms/formio/components/np_family_members/tests/test_family_members.py b/src/openforms/formio/components/np_family_members/tests/test_family_members.py index 9e8f20a6c6..12d6b7eff0 100644 --- a/src/openforms/formio/components/np_family_members/tests/test_family_members.py +++ b/src/openforms/formio/components/np_family_members/tests/test_family_members.py @@ -24,12 +24,11 @@ class FamilyMembersCustomFieldTypeTest(TestCase): - @patch("openforms.formio.components.custom.NPFamilyMembers._get_handler") - def test_get_values_for_custom_field(self, mock_get_handler): - mock_get_handler.return_value.return_value = [ - ("222333444", "Billy Doe"), - ("333444555", "Jane Doe"), - ] + @patch( + "openforms.formio.components.custom.get_np_children_haal_centraal", + return_value=[("222333444", "Billy Doe"), ("333444555", "Jane Doe")], + ) + def test_get_values_for_custom_field(self, mock_get_np_children): submission = SubmissionFactory.from_components( [ { diff --git a/src/openforms/formio/datastructures.py b/src/openforms/formio/datastructures.py index 2385ed9266..eb3e2e467a 100644 --- a/src/openforms/formio/datastructures.py +++ b/src/openforms/formio/datastructures.py @@ -52,14 +52,3 @@ def __add__( @property def configuration(self) -> JSONObject: return self._configuration - - @configuration.setter - def configuration(self, new: JSONObject) -> None: - # if there are no changes in the configuration, do not invalidate the cache - if self._configuration == new: - return - - # invalidate cache on configuration mutations - if self._cached_component_map is not None: - self._cached_component_map = None - self._configuration = new diff --git a/src/openforms/formio/dynamic_config/tests/test_file_component.py b/src/openforms/formio/dynamic_config/tests/test_file_component.py new file mode 100644 index 0000000000..32bebd6747 --- /dev/null +++ b/src/openforms/formio/dynamic_config/tests/test_file_component.py @@ -0,0 +1,41 @@ +from unittest.mock import patch + +from django.test import TestCase + +from rest_framework.test import APIRequestFactory + +from openforms.config.models import GlobalConfiguration + +from ...datastructures import FormioConfigurationWrapper +from ...service import rewrite_formio_components_for_request + +request_factory = APIRequestFactory() + + +def _get_dynamic_config(component: dict) -> FormioConfigurationWrapper: + wrapper = FormioConfigurationWrapper({"components": [component]}) + request = request_factory.get("/irrelevant") + return rewrite_formio_components_for_request(wrapper, request) + + +@patch( + "openforms.formio.components.vanilla.GlobalConfiguration.get_solo", + return_value=GlobalConfiguration( + form_upload_default_file_types=["image/png", "application/pdf"] + ), +) +class FileComponentTests(TestCase): + def test_use_global_config_filetypes(self, m_get_solo): + component = { + "type": "file", + "key": "fileTest", + "url": "", + "useConfigFiletypes": True, + "filePattern": "*", + } + + wrapper = _get_dynamic_config(component) + + updated_component = wrapper["fileTest"] + self.assertEqual(updated_component["filePattern"], "image/png,application/pdf") + self.assertNotEqual(updated_component["url"], "") diff --git a/src/openforms/formio/formatters/base.py b/src/openforms/formio/formatters/base.py index 91964b6292..4239564cbe 100644 --- a/src/openforms/formio/formatters/base.py +++ b/src/openforms/formio/formatters/base.py @@ -70,5 +70,5 @@ def __call__(self, component: Component, value: Any) -> str: component, self.join_formatted_values(component, formatted_values) ) - def format(self, component: Component, value: Any) -> str: + def format(self, component: Component, value: Any) -> str: # pragma:nocover raise NotImplementedError("%r must implement the 'format' method" % type(self)) diff --git a/src/openforms/formio/formatters/service.py b/src/openforms/formio/formatters/service.py deleted file mode 100644 index 3bc22580f4..0000000000 --- a/src/openforms/formio/formatters/service.py +++ /dev/null @@ -1,15 +0,0 @@ -import warnings - -from ..service import format_value as _format_value -from .printable import filter_printable - -__all__ = ["filter_printable"] - - -def format_value(*args, **kwargs): - warnings.warn( - "`openforms.formio.formatters.service.format_value` is deprecated in favour of " - "`openforms.formio.service.format_value`. Please update your references.", - DeprecationWarning, - ) - return _format_value(*args, **kwargs) diff --git a/src/openforms/formio/registry.py b/src/openforms/formio/registry.py index a2b84d6f06..1a3c5f40ed 100644 --- a/src/openforms/formio/registry.py +++ b/src/openforms/formio/registry.py @@ -23,11 +23,11 @@ from .typing import Component -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: nocover from openforms.submissions.models import Submission -class FormatterProtocol(Protocol): +class FormatterProtocol(Protocol): # pragma: nocover def __init__(self, as_html: bool): ... @@ -35,12 +35,12 @@ def __call__(self, component: Component, value: Any) -> str: ... -class NormalizerProtocol(Protocol): +class NormalizerProtocol(Protocol): # pragma: nocover def __call__(self, component: Component, value: Any) -> Any: ... -class RewriterForRequestProtocol(Protocol): +class RewriterForRequestProtocol(Protocol): # pragma: nocover def __call__(self, component: Component, request: Request) -> None: ... diff --git a/src/openforms/formio/tests/test_normalization.py b/src/openforms/formio/tests/test_normalization.py index 1509aa20f6..a3e8f2fc6f 100644 --- a/src/openforms/formio/tests/test_normalization.py +++ b/src/openforms/formio/tests/test_normalization.py @@ -56,3 +56,10 @@ def test_no_input_mask_given(self): result = normalize_value_for_component(component, "AAAA 34") self.assertEqual(result, "AAAA 34") + + def test_normalize_unknown_component_type(self): + component = {"type": "7923abf1-1397-40ed-b194-7a1d05e23b23"} + + result = normalize_value_for_component(component, "foo.bar-baz") + + self.assertEqual(result, "foo.bar-baz") # unmodified diff --git a/src/openforms/submissions/models/submission.py b/src/openforms/submissions/models/submission.py index 2526d8325b..505df43b49 100644 --- a/src/openforms/submissions/models/submission.py +++ b/src/openforms/submissions/models/submission.py @@ -507,7 +507,7 @@ def get_last_completed_step(self) -> Optional["SubmissionStep"]: return submission_state.get_last_completed_step() def get_ordered_data_with_component_type(self) -> OrderedDict: - from openforms.formio.formatters.service import filter_printable + from openforms.formio.formatters.printable import filter_printable ordered_data = OrderedDict() merged_data = self.get_merged_data()