From c7d7f932576eef3e6a954ef05b57bbdff978ae07 Mon Sep 17 00:00:00 2001 From: Viktor van Wijk Date: Tue, 4 Feb 2025 17:08:35 +0100 Subject: [PATCH] :bug: [#5065] Add processing of edit grid component in JSON dump plugin The edit grid component was not processed, meaning components (File, Radio, Select, and SelectBoxes) inside the edit grid component were not updated properly. They are now --- .../registrations/contrib/json_dump/plugin.py | 224 ++++++++++++------ 1 file changed, 155 insertions(+), 69 deletions(-) diff --git a/src/openforms/registrations/contrib/json_dump/plugin.py b/src/openforms/registrations/contrib/json_dump/plugin.py index b377d0f6a4..339329a8ad 100644 --- a/src/openforms/registrations/contrib/json_dump/plugin.py +++ b/src/openforms/registrations/contrib/json_dump/plugin.py @@ -1,15 +1,19 @@ import base64 import json +from collections import defaultdict from typing import cast from django.core.exceptions import SuspiciousOperation from django.core.serializers.json import DjangoJSONEncoder +from django.db.models import F, TextField, Value +from django.db.models.functions import Coalesce, NullIf from django.utils.translation import gettext_lazy as _ from zgw_consumers.client import build_client from openforms.formio.service import rewrite_formio_components from openforms.formio.typing import ( + Component, FileComponent, RadioComponent, SelectBoxesComponent, @@ -79,10 +83,11 @@ def post_process( ) -> None: """Post-process the values and schema. - File components need special treatment, as we send the content of the file - encoded with base64, instead of the output from the serializer. Also, Radio, - Select, and SelectBoxes components need to be updated if their data source is - set to another form variable. + - Update the configuration wrapper. This is necessary to update the options for + Select, SelectBoxes, and Radio components that get their options from another form + variable. + - Get all attachments of this submission, and group them by the data path + - Process each component :param values: JSONObject :param schema: JSONObject @@ -90,9 +95,7 @@ def post_process( """ state = submission.load_submission_value_variables_state() - # Update config wrapper. This is necessary to update the options for Select, - # SelectBoxes, and Radio components that get their options from another form - # variable. + # Update config wrapper data = DataContainer(state=state) configuration_wrapper = rewrite_formio_components( submission.total_configuration_wrapper, @@ -100,6 +103,24 @@ def post_process( data=data.data, ) + # Create attachment mapping from key or component data path to attachment list + attachments = submission.attachments.annotate( + data_path=Coalesce( + NullIf( + F("_component_data_path"), + Value(""), + ), + # fall back to variable/component key if no explicit data path is set + F("submission_variable__key"), + output_field=TextField(), + ) + ) + attachments_dict = defaultdict(list) + for attachment in attachments: + attachments_dict[attachment.data_path].append( + attachment + ) # pyright: ignore[reportAttributeAccessIssue] + for key in values.keys(): variable = state.variables.get(key) if ( @@ -113,86 +134,151 @@ def post_process( component = configuration_wrapper[key] assert component is not None - match component: - case {"type": "file", "multiple": True}: - attachment_list, base_schema = get_attachments_and_base_schema( - cast(FileComponent, component), submission - ) - values[key] = attachment_list # type: ignore - schema["properties"][key] = to_multiple(base_schema) # type: ignore - - case {"type": "file"}: # multiple is False or missing - attachment_list, base_schema = get_attachments_and_base_schema( - cast(FileComponent, component), submission - ) - - assert (n_attachments := len(attachment_list)) <= 1 # sanity check - if n_attachments == 0: - value = None - base_schema = {"title": component["label"], "type": "null"} - else: - value = attachment_list[0] - values[key] = value - schema["properties"][key] = base_schema # type: ignore - - case {"type": "radio", "openForms": {"dataSrc": "variable"}}: - component = cast(RadioComponent, component) - choices = [options["value"] for options in component["values"]] - choices.append("") # Take into account an unfilled field + process_component( + component, values, schema, attachments_dict, configuration_wrapper + ) + + +def process_component( + component: Component, + values: JSONObject, + schema: JSONObject, + attachments: dict[str, list[SubmissionFileAttachment]], + configuration_wrapper, + key_prefix: str = "", +) -> None: + """Process a component. + + The following components needs extra attention: + - File components: we send the content of the file encoded with base64, instead of + the output from the serializer. + - Radio, Select, and SelectBoxes components: need to be updated if their data source + is set to another form variable. + - Edit grid components: layout component with other components as children, which + (potentially) need to be processed. + + :param component: Component + :param values: JSONObject + :param schema: JSONObject + :param attachments: Mapping from component submission data path to list of attachments + corresponding to that component. + :param configuration_wrapper: Updated total configuration wrapper. + :param key_prefix: If the component is part of an edit grid component, this key + prefix includes the parent key and the index of the component as it appears in the + submitted data list of that edit grid component. + """ + key = component["key"] + + match component: + case {"type": "file", "multiple": True}: + attachment_list, base_schema = get_attachments_and_base_schema( + cast(FileComponent, component), attachments, key_prefix + ) + values[key] = attachment_list # type: ignore + schema["properties"][key] = to_multiple(base_schema) # type: ignore + + case {"type": "file"}: # multiple is False or missing + attachment_list, base_schema = get_attachments_and_base_schema( + cast(FileComponent, component), attachments, key_prefix + ) + + assert (n_attachments := len(attachment_list)) <= 1 # sanity check + if n_attachments == 0: + value = None + base_schema = {"title": component["label"], "type": "null"} + else: + value = attachment_list[0] + values[key] = value + schema["properties"][key] = base_schema # type: ignore + + case {"type": "radio", "openForms": {"dataSrc": "variable"}}: + component = cast(RadioComponent, component) + choices = [options["value"] for options in component["values"]] + choices.append("") # Take into account an unfilled field + schema["properties"][key]["enum"] = choices # type: ignore + + case {"type": "select", "openForms": {"dataSrc": "variable"}}: + component = cast(SelectComponent, component) + choices = [options["value"] for options in component["data"]["values"]] # type: ignore[reportTypedDictNotRequiredAccess] + choices.append("") # Take into account an unfilled field + + if component.get("multiple", False): + schema["properties"][key]["items"]["enum"] = choices # type: ignore + else: schema["properties"][key]["enum"] = choices # type: ignore - case {"type": "select", "openForms": {"dataSrc": "variable"}}: - component = cast(SelectComponent, component) - choices = [options["value"] for options in component["data"]["values"]] # type: ignore[reportTypedDictNotRequiredAccess] - choices.append("") # Take into account an unfilled field - - if component.get("multiple", False): - schema["properties"][key]["items"]["enum"] = choices # type: ignore - else: - schema["properties"][key]["enum"] = choices # type: ignore - - case {"type": "selectboxes"}: - component = cast(SelectBoxesComponent, component) - data_src = component.get("openForms", {}).get("dataSrc") - - if data_src == "variable": - properties = { - options["value"]: {"type": "boolean"} - for options in component["values"] - } - base_schema = { - "properties": properties, - "required": list(properties.keys()), - "additionalProperties": False, - } - schema["properties"][key].update(base_schema) # type: ignore - - # If the select boxes component was hidden, the submitted data of this - # component is an empty dict, so set the required to an empty list. - if not values[key]: - schema["properties"][key]["required"] = [] # type: ignore - case _: - pass + case {"type": "selectboxes"}: + component = cast(SelectBoxesComponent, component) + data_src = component.get("openForms", {}).get("dataSrc") + + if data_src == "variable": + properties = { + options["value"]: {"type": "boolean"} + for options in component["values"] + } + base_schema = { + "properties": properties, + "required": list(properties.keys()), + "additionalProperties": False, + } + schema["properties"][key].update(base_schema) # type: ignore + + # If the select boxes component was hidden, the submitted data of this + # component is an empty dict, so set the required to an empty list. + if not values[key]: + schema["properties"][key]["required"] = [] # type: ignore + + case {"type": "editgrid"}: + # Note: the schema actually only needs to be processed once for each child + # component, but will be processed for each repeating group entry for + # implementation simplicity. + edit_grid_schema = schema["properties"][key]["items"] + + for index, edit_grid_values in enumerate(values[key]): + + for child_key in edit_grid_values.keys(): + process_component( + component=configuration_wrapper[child_key], + values=edit_grid_values, + schema=edit_grid_schema, + attachments=attachments, + configuration_wrapper=configuration_wrapper, + key_prefix=( + f"{key_prefix}.{key}.{index}" + if key_prefix + else f"{key}.{index}" + ), + ) + + case _: + pass def get_attachments_and_base_schema( - component: FileComponent, submission: Submission + component: FileComponent, + attachments: dict[str, list[SubmissionFileAttachment]], + key_prefix: str = "", ) -> tuple[list[JSONObject], JSONObject]: """Return list of encoded attachments and the base schema. :param component: FileComponent - :param submission: Submission + :param attachments: Mapping from component submission data path to list of attachments + corresponding to that component. + :param key_prefix: If the file component is part of an edit grid component, this key + prefix includes the parent key and the index of the component as it appears in the + submitted data list of the edit grid component. :return encoded_attachments: list[JSONObject] :return base_schema: JSONObject """ + key = f"{key_prefix}.{component['key']}" if key_prefix else component["key"] + encoded_attachments: list[JSONObject] = [ { "file_name": attachment.original_name, "content": encode_attachment(attachment), } - for attachment in submission.attachments - if attachment.form_key == component["key"] + for attachment in attachments.get(key, []) ] base_schema: JSONObject = {