Skip to content

Commit

Permalink
🐛 [#4689] Fix uploads in repeating groups not being processed correctly
Browse files Browse the repository at this point in the history 
We must look up the uploads in the url map and replace only the
file component nodes in the item values rather than the whole
repeating group. This also needs to recurse, since in theory
the repeating group can have another repeating group inside
it.
  • Loading branch information
@sergei-maertens
sergei-maertens committed Jan 30, 2025
1 parent fb63835 commit 2ae1a36
Show file tree
Hide file tree
Showing 9 changed files with 226 additions and 139 deletions.
101 changes: 86 additions & 15 deletions src/openforms/registrations/contrib/objects_api/handlers/v2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@
from glom import Assign, Path, glom

from openforms.api.utils import underscore_to_camel
from openforms.formio.typing import Component
from openforms.formio.service import FormioData
from openforms.formio.typing import Component, EditGridComponent
from openforms.formio.typing.vanilla import FileComponent
from openforms.typing import JSONObject, JSONValue

from ..typing import ObjecttypeVariableMapping
Expand Down Expand Up @@ -81,7 +83,7 @@ def process_mapped_variable(
which value needs to be written to which "object path" for the record data, for
possible deep assignments.
"""

variable_key = mapping["variable_key"]
target_path = Path(*bits) if (bits := mapping.get("target_path")) else None

# normalize non-primitive date/datetime values so that they're ready for JSON
Expand Down Expand Up @@ -120,27 +122,96 @@ def process_mapped_variable(
assert target_path is not None
return AssignmentSpec(destination=target_path, value=value)

case {"type": "file", **rest}:
case {"type": "file"}:
assert attachment_urls is not None
multiple = rest.get("multiple", False)
upload_urls = attachment_urls[mapping["variable_key"]]

transformed_value = str | list[str]

# for multiple uploads, replace the Formio file dicts with our upload URLs
if multiple:
transformed_value = upload_urls
# single file - return only one element *if* there are uploads
else:
transformed_value = upload_urls[0] if upload_urls else ""
value = cast(JSONValue, transformed_value)
value = _transform_file_value(
cast(FileComponent, component), attachment_urls
)

case {"type": "map"}:
assert isinstance(value, dict)

case {"type": "editgrid"} if attachment_urls is not None:
assert isinstance(value, list)
value = _transform_editgrid_value(
cast(EditGridComponent, component),
cast(list[JSONObject], value),
attachment_urls=attachment_urls,
key_prefix=variable_key,
)
# not a component or standard behaviour where no transformation is necessary
case None | _:
pass

assert target_path is not None
return AssignmentSpec(destination=target_path, value=value)


def _transform_file_value(
component: FileComponent,
attachment_urls: dict[str, list[str]],
key_prefix: str = "",
) -> str | list[str]:
"""
Transform a single file component value according to the component configuration.
"""
key = component["key"]
multiple = component.get("multiple", False)

# it's possible keys are missing because there are no uploads at all for the
# component.
data_path = f"{key_prefix}.{key}" if key_prefix else key
upload_urls = attachment_urls.get(data_path, [])

match upload_urls:
# if there are no uploads and it's a single component -> normalize to empty string
case [] if not multiple:
return ""

# if there's an upload and it's a single component -> return the single URL string
case list() if upload_urls and not multiple:
return upload_urls[0]

# otherwise just return the list of upload URLs
case list():
assert multiple
return upload_urls


def _transform_editgrid_value(
component: EditGridComponent,
value: list[JSONObject],
attachment_urls: dict[str, list[str]],
key_prefix: str,
) -> list[JSONObject]:
nested_components = component["components"]

items: list[JSONObject] = []

# process file uploads inside (nested) repeating groups
for index, item in enumerate(value):
item_values = FormioData(item)

for nested_component in nested_components:
key = nested_component["key"]

match nested_component:
case {"type": "file"}:
item_values[key] = _transform_file_value(
cast(FileComponent, nested_component),
attachment_urls=attachment_urls,
key_prefix=f"{key_prefix}.{index}",
)
case {"type": "editgrid"}:
nested_items = item_values[key]
assert isinstance(nested_items, list)
item_values[key] = _transform_editgrid_value(
cast(EditGridComponent, nested_component),
value=cast(list[JSONObject], nested_items),
attachment_urls=attachment_urls,
key_prefix=f"{key_prefix}.{index}.{key}",
)

items.append(item_values.data)

return items
10 changes: 5 additions & 5 deletions ...ObjectsAPIBackendV2Tests/ObjectsAPIBackendV2Tests.test_addressNl_legacy_before_of_30.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ interactions:
everything","namePlural":"Accepts everything","description":"","dataClassification":"open","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-07-22","modifiedAt":"2024-07-22","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/8faed0fa-7864-4409-aa6d-533a37616a9e/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/644ab597-e88c-43c0-8321-f12113510b0e","uuid":"644ab597-e88c-43c0-8321-f12113510b0e","name":"Fieldset
component","namePlural":"Fieldset component","description":"","dataClassification":"confidential","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-02-08","modifiedAt":"2024-02-08","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/644ab597-e88c-43c0-8321-f12113510b0e/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/f1dde4fe-b7f9-46dc-84ae-429ae49e3705","uuid":"f1dde4fe-b7f9-46dc-84ae-429ae49e3705","name":"Geo
in data","namePlural":"Geo in data","description":"","dataClassification":"confidential","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-02-08","modifiedAt":"2024-02-08","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/f1dde4fe-b7f9-46dc-84ae-429ae49e3705/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/527b8408-7421-4808-a744-43ccb7bdaaa2","uuid":"527b8408-7421-4808-a744-43ccb7bdaaa2","name":"File
Uploads","namePlural":"File Uploads","description":"","dataClassification":"confidential","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-02-08","modifiedAt":"2024-02-08","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/527b8408-7421-4808-a744-43ccb7bdaaa2/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/3edfdaf7-f469-470b-a391-bb7ea015bd6f","uuid":"3edfdaf7-f469-470b-a391-bb7ea015bd6f","name":"Tree","namePlural":"Trees","description":"","dataClassification":"confidential","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-02-08","modifiedAt":"2024-02-08","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/3edfdaf7-f469-470b-a391-bb7ea015bd6f/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","uuid":"8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","name":"Person","namePlural":"Persons","description":"","dataClassification":"open","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2023-10-24","modifiedAt":"2024-11-25","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/1","http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/2","http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/3","http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/4"]}]}'
Uploads","namePlural":"File Uploads","description":"","dataClassification":"confidential","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-02-08","modifiedAt":"2024-02-08","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/527b8408-7421-4808-a744-43ccb7bdaaa2/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/3edfdaf7-f469-470b-a391-bb7ea015bd6f","uuid":"3edfdaf7-f469-470b-a391-bb7ea015bd6f","name":"Tree","namePlural":"Trees","description":"","dataClassification":"confidential","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2024-02-08","modifiedAt":"2024-02-08","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/3edfdaf7-f469-470b-a391-bb7ea015bd6f/versions/1"]},{"url":"http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","uuid":"8e46e0a5-b1b4-449b-b9e9-fa3cea655f48","name":"Person","namePlural":"Persons","description":"","dataClassification":"open","maintainerOrganization":"","maintainerDepartment":"","contactPerson":"","contactEmail":"","source":"","updateFrequency":"unknown","providerOrganization":"","documentationUrl":"","labels":{},"createdAt":"2023-10-24","modifiedAt":"2024-11-25","allowGeometry":true,"versions":["http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/2","http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/3","http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/1","http://objecttypes-web:8000/api/v2/objecttypes/8e46e0a5-b1b4-449b-b9e9-fa3cea655f48/versions/4"]}]}'
headers:
Allow:
- GET, POST, HEAD, OPTIONS
Expand All @@ -33,11 +33,11 @@ interactions:
Cross-Origin-Opener-Policy:
- same-origin
Date:
- Thu, 19 Dec 2024 08:19:29 GMT
- Wed, 29 Jan 2025 17:01:16 GMT
Referrer-Policy:
- same-origin
Server:
- nginx/1.27.0
- nginx/1.27.3
Vary:
- origin
X-Content-Type-Options:
Expand Down Expand Up @@ -77,11 +77,11 @@ interactions:
Cross-Origin-Opener-Policy:
- same-origin
Date:
- Thu, 19 Dec 2024 08:19:29 GMT
- Wed, 29 Jan 2025 17:01:16 GMT
Referrer-Policy:
- same-origin
Server:
- nginx/1.27.0
- nginx/1.27.3
Vary:
- origin
X-Content-Type-Options:
Expand Down
35 changes: 18 additions & 17 deletions ...kendV2Tests/ObjectsAPIBackendV2Tests.test_submission_with_editgrid_with_nested_files.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ interactions:
- request:
body: '{"informatieobjecttype": "http://localhost:8003/catalogi/api/v1/informatieobjecttypen/531f6c1a-97f7-478c-85f0-67d2f23661c7",
"bronorganisatie": "000000000", "creatiedatum": "2024-03-19", "titel": "Form
001", "auteur": "Aanvrager", "taal": "nld", "formaat": "application/ogg", "inhoud":
"Y29udGVudA==", "status": "definitief", "bestandsnaam": "foo-0.bin", "ontvangstdatum":
"2024-03-06", "beschrijving": "Bijgevoegd document", "indicatieGebruiksrecht":
002", "auteur": "Aanvrager", "taal": "nld", "formaat": "application/octet-stream",
"inhoud": "Y29udGVudA==", "status": "definitief", "bestandsnaam": "foo-0.bin",
"ontvangstdatum": "2024-03-17", "beschrijving": "Bijgevoegd document", "indicatieGebruiksrecht":
false, "bestandsomvang": 7}'
headers:
Accept:
Expand All @@ -16,7 +16,7 @@ interactions:
Connection:
- keep-alive
Content-Length:
- '479'
- '488'
Content-Type:
- application/json
User-Agent:
Expand All @@ -25,22 +25,22 @@ interactions:
uri: http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten
response:
body:
string: '{"url":"http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/db57bb11-cd3d-4a0e-8784-9ac308f5ab7b","identificatie":"DOCUMENT-2024-0000000094","bronorganisatie":"000000000","creatiedatum":"2024-03-19","titel":"Form
001","vertrouwelijkheidaanduiding":"openbaar","auteur":"Aanvrager","status":"definitief","formaat":"application/ogg","taal":"nld","versie":1,"beginRegistratie":"2025-01-29T13:43:03.815170Z","bestandsnaam":"foo-0.bin","inhoud":"http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/db57bb11-cd3d-4a0e-8784-9ac308f5ab7b/download?versie=1","bestandsomvang":7,"link":"","beschrijving":"Bijgevoegd
document","ontvangstdatum":"2024-03-06","verzenddatum":null,"indicatieGebruiksrecht":false,"verschijningsvorm":"","ondertekening":{"soort":"","datum":null},"integriteit":{"algoritme":"","waarde":"","datum":null},"informatieobjecttype":"http://localhost:8003/catalogi/api/v1/informatieobjecttypen/531f6c1a-97f7-478c-85f0-67d2f23661c7","locked":false,"bestandsdelen":[],"trefwoorden":[],"lock":""}'
string: '{"url":"http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/4b11e35f-5d76-48c8-a7dc-91b1d8086969","identificatie":"DOCUMENT-2024-0000000165","bronorganisatie":"000000000","creatiedatum":"2024-03-19","titel":"Form
002","vertrouwelijkheidaanduiding":"openbaar","auteur":"Aanvrager","status":"definitief","formaat":"application/octet-stream","taal":"nld","versie":1,"beginRegistratie":"2025-01-29T17:01:16.380237Z","bestandsnaam":"foo-0.bin","inhoud":"http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/4b11e35f-5d76-48c8-a7dc-91b1d8086969/download?versie=1","bestandsomvang":7,"link":"","beschrijving":"Bijgevoegd
document","ontvangstdatum":"2024-03-17","verzenddatum":null,"indicatieGebruiksrecht":false,"verschijningsvorm":"","ondertekening":{"soort":"","datum":null},"integriteit":{"algoritme":"","waarde":"","datum":null},"informatieobjecttype":"http://localhost:8003/catalogi/api/v1/informatieobjecttypen/531f6c1a-97f7-478c-85f0-67d2f23661c7","locked":false,"bestandsdelen":[],"trefwoorden":[],"lock":""}'
headers:
API-version:
- 1.4.2
Allow:
- GET, POST, HEAD, OPTIONS
Content-Length:
- '1035'
- '1044'
Content-Type:
- application/json
Cross-Origin-Opener-Policy:
- same-origin
Location:
- http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/db57bb11-cd3d-4a0e-8784-9ac308f5ab7b
- http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/4b11e35f-5d76-48c8-a7dc-91b1d8086969
Referrer-Policy:
- same-origin
Vary:
Expand Down Expand Up @@ -83,7 +83,7 @@ interactions:
Cross-Origin-Opener-Policy:
- same-origin
Date:
- Wed, 29 Jan 2025 13:43:03 GMT
- Wed, 29 Jan 2025 17:01:16 GMT
Referrer-Policy:
- same-origin
Server:
Expand All @@ -99,8 +99,9 @@ interactions:
message: OK
- request:
body: '{"type": "http://objecttypes-web:8000/api/v2/objecttypes/8faed0fa-7864-4409-aa6d-533a37616a9e",
"record": {"typeVersion": 1, "data": {"repeatingGroup": ["http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/db57bb11-cd3d-4a0e-8784-9ac308f5ab7b"]},
"startAt": "2024-03-19"}}'
"record": {"typeVersion": 1, "data": {"repeatingGroup": [{"nestedRepeatingGroup":
[{"file": "http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/4b11e35f-5d76-48c8-a7dc-91b1d8086969",
"email": "info@example.com"}]}]}, "startAt": "2024-03-19"}}'
headers:
Accept:
- '*/*'
Expand All @@ -113,7 +114,7 @@ interactions:
Content-Crs:
- EPSG:4326
Content-Length:
- '290'
- '357'
Content-Type:
- application/json
User-Agent:
Expand All @@ -122,7 +123,7 @@ interactions:
uri: http://localhost:8002/api/v2/objects
response:
body:
string: '{"url":"http://objects-web:8000/api/v2/objects/75f65dae-4711-4bb6-95cf-513b709ec401","uuid":"75f65dae-4711-4bb6-95cf-513b709ec401","type":"http://objecttypes-web:8000/api/v2/objecttypes/8faed0fa-7864-4409-aa6d-533a37616a9e","record":{"index":1,"typeVersion":1,"data":{"repeatingGroup":["http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/db57bb11-cd3d-4a0e-8784-9ac308f5ab7b"]},"geometry":null,"startAt":"2024-03-19","endAt":null,"registrationAt":"2025-01-29","correctionFor":null,"correctedBy":null}}'
string: '{"url":"http://objects-web:8000/api/v2/objects/35ccae85-b9a1-4b1b-8677-8ae9404ba4a2","uuid":"35ccae85-b9a1-4b1b-8677-8ae9404ba4a2","type":"http://objecttypes-web:8000/api/v2/objecttypes/8faed0fa-7864-4409-aa6d-533a37616a9e","record":{"index":1,"typeVersion":1,"data":{"repeatingGroup":[{"nestedRepeatingGroup":[{"file":"http://localhost:8003/documenten/api/v1/enkelvoudiginformatieobjecten/4b11e35f-5d76-48c8-a7dc-91b1d8086969","email":"info@example.com"}]}]},"geometry":null,"startAt":"2024-03-19","endAt":null,"registrationAt":"2025-01-29","correctionFor":null,"correctedBy":null}}'
headers:
Allow:
- GET, POST, HEAD, OPTIONS
Expand All @@ -131,15 +132,15 @@ interactions:
Content-Crs:
- EPSG:4326
Content-Length:
- '520'
- '583'
Content-Type:
- application/json
Cross-Origin-Opener-Policy:
- same-origin
Date:
- Wed, 29 Jan 2025 13:43:03 GMT
- Wed, 29 Jan 2025 17:01:16 GMT
Location:
- http://localhost:8002/api/v2/objects/75f65dae-4711-4bb6-95cf-513b709ec401
- http://localhost:8002/api/v2/objects/35ccae85-b9a1-4b1b-8677-8ae9404ba4a2
Referrer-Policy:
- same-origin
Server:
Expand Down
12 changes: 6 additions & 6 deletions ...sAPIBackendV2Tests/ObjectsAPIBackendV2Tests.test_submission_with_empty_optional_file.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ interactions:
Cross-Origin-Opener-Policy:
- same-origin
Date:
- Thu, 19 Dec 2024 08:19:30 GMT
- Wed, 29 Jan 2025 17:01:16 GMT
Referrer-Policy:
- same-origin
Server:
- nginx/1.27.0
- nginx/1.27.3
Vary:
- origin
X-Content-Type-Options:
Expand Down Expand Up @@ -68,7 +68,7 @@ interactions:
uri: http://localhost:8002/api/v2/objects
response:
body:
string: '{"url":"http://objects-web:8000/api/v2/objects/c277e6c8-02da-4257-9051-a13bb67b3d4e","uuid":"c277e6c8-02da-4257-9051-a13bb67b3d4e","type":"http://objecttypes-web:8000/api/v2/objecttypes/527b8408-7421-4808-a744-43ccb7bdaaa2","record":{"index":1,"typeVersion":1,"data":{"single_file":""},"geometry":null,"startAt":"2024-03-19","endAt":null,"registrationAt":"2024-12-19","correctionFor":null,"correctedBy":null}}'
string: '{"url":"http://objects-web:8000/api/v2/objects/28bb7db5-2806-4291-a731-68ffa0829977","uuid":"28bb7db5-2806-4291-a731-68ffa0829977","type":"http://objecttypes-web:8000/api/v2/objecttypes/527b8408-7421-4808-a744-43ccb7bdaaa2","record":{"index":1,"typeVersion":1,"data":{"single_file":""},"geometry":null,"startAt":"2024-03-19","endAt":null,"registrationAt":"2025-01-29","correctionFor":null,"correctedBy":null}}'
headers:
Allow:
- GET, POST, HEAD, OPTIONS
Expand All @@ -83,13 +83,13 @@ interactions:
Cross-Origin-Opener-Policy:
- same-origin
Date:
- Thu, 19 Dec 2024 08:19:30 GMT
- Wed, 29 Jan 2025 17:01:16 GMT
Location:
- http://localhost:8002/api/v2/objects/c277e6c8-02da-4257-9051-a13bb67b3d4e
- http://localhost:8002/api/v2/objects/28bb7db5-2806-4291-a731-68ffa0829977
Referrer-Policy:
- same-origin
Server:
- nginx/1.27.0
- nginx/1.27.3
Vary:
- origin
X-Content-Type-Options:
Expand Down
Loading

0 comments on commit 2ae1a36

Please sign in to comment.