diff --git a/CHANGES/672.feature b/CHANGES/672.feature new file mode 100644 index 000000000..baed9b429 --- /dev/null +++ b/CHANGES/672.feature @@ -0,0 +1 @@ +Added validation for uploaded manifest JSON content. diff --git a/CHANGES/853.bugfix b/CHANGES/853.bugfix new file mode 100644 index 000000000..1a9153b84 --- /dev/null +++ b/CHANGES/853.bugfix @@ -0,0 +1,2 @@ +Fixed internal server errors raised when a podman client (<4.0) used invalid content types for +manifest lists. diff --git a/CHANGES/854.bugfix b/CHANGES/854.bugfix new file mode 100644 index 000000000..00e3aeb5d --- /dev/null +++ b/CHANGES/854.bugfix @@ -0,0 +1 @@ +Fixed a misleading error message raised when a user provided an invalid manifest list. diff --git a/pulp_container/app/json_schemas.py b/pulp_container/app/json_schemas.py index ca0ce7e01..3dce83e57 100644 --- a/pulp_container/app/json_schemas.py +++ b/pulp_container/app/json_schemas.py @@ -1,6 +1,219 @@ -SIGNATURE_SCHEMA = """{ - "$schema": "http://json-schema.org/draft-07/schema", - "$id": "https://example.com/product.schema.json", +def get_descriptor_schema( + allowed_media_types, additional_properties=None, additional_required=None +): + """Return a concrete descriptor schema for manifests.""" + properties = { + "mediaType": {"type": "string", "enum": allowed_media_types}, + "size": {"type": "number"}, + "digest": {"type": "string"}, + "annotations": {"type": "object", "additionalProperties": True}, + "urls": {"type": "array", "items": {"type": "string"}}, + } + + if additional_properties: + properties.update(additional_properties) + + required = ["mediaType", "size", "digest"] + if additional_required: + required.extend(additional_required) + + return {"type": "object", "properties": properties, "required": required} + + +OCI_INDEX_SCHEMA = { + "type": "object", + "properties": { + "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2}, + "mediaType": { + "type": "string", + "enum": ["application/vnd.oci.image.index.v1+json"], + }, + "manifests": { + "type": "array", + "items": get_descriptor_schema( + allowed_media_types=[ + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.oci.image.index.v1+json", + ], + additional_properties={ + "platform": { + "type": "object", + "properties": { + "architecture": {"type": "string"}, + "os": {"type": "string"}, + "os.version": {"type": "string"}, + "os.features": {"type": "array", "items": {"type": "string"}}, + "variant": {"type": "string"}, + "features": {"type": "array", "items": {"type": "string"}}, + }, + "required": ["architecture", "os"], + }, + }, + additional_required=["platform"], + ), + }, + "annotations": {"type": "object", "additionalProperties": True}, + }, + "required": ["schemaVersion", "manifests"], +} + +OCI_MANIFEST_SCHEMA = { + "type": "object", + "properties": { + "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2}, + "mediaType": { + "type": "string", + "enum": ["application/vnd.oci.image.manifest.v1+json"], + }, + "config": get_descriptor_schema(["application/vnd.oci.image.config.v1+json"]), + "layers": { + "type": "array", + "items": get_descriptor_schema( + [ + "application/vnd.oci.image.layer.v1.tar", + "application/vnd.oci.image.layer.v1.tar+gzip", + "application/vnd.oci.image.layer.v1.tar+zstd", + "application/vnd.oci.image.layer.nondistributable.v1.tar", + "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", + ] + ), + }, + }, + "required": ["schemaVersion", "config", "layers"], +} + +DOCKER_MANIFEST_LIST_V2_SCHEMA = { + "type": "object", + "properties": { + "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2}, + "mediaType": { + "type": "string", + "enum": ["application/vnd.docker.distribution.manifest.list.v2+json"], + }, + "manifests": { + "type": "array", + "items": { + "type": "object", + "properties": { + "mediaType": { + "type": "string", + "enum": [ + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.docker.distribution.manifest.v1+json", + ], + }, + "size": {"type": "number"}, + "digest": {"type": "string"}, + "platform": { + "type": "object", + "properties": { + "architecture": {"type": "string"}, + "os": {"type": "string"}, + "os.version": {"type": "string"}, + "os.features": { + "type": "array", + "items": {"type": "string"}, + }, + "variant": {"type": "string"}, + "features": { + "type": "array", + "items": {"type": "string"}, + }, + }, + "required": ["architecture", "os"], + }, + }, + "required": ["mediaType", "size", "digest", "platform"], + }, + }, + }, + "required": ["schemaVersion", "mediaType", "manifests"], +} + +DOCKER_MANIFEST_V2_SCHEMA = { + "type": "object", + "properties": { + "schemaVersion": {"type": "number", "minimum": 2, "maximum": 2}, + "mediaType": { + "type": "string", + "enum": ["application/vnd.docker.distribution.manifest.v2+json"], + }, + "config": { + "type": "object", + "properties": { + "mediaType": { + "type": "string", + "enum": ["application/vnd.docker.container.image.v1+json"], + }, + "size": {"type": "number"}, + "digest": {"type": "string"}, + }, + "required": ["mediaType", "size", "digest"], + }, + "layers": { + "type": "array", + "items": { + "type": "object", + "properties": { + "mediaType:": { + "type": "string", + "enum": [ + "application/vnd.docker.image.rootfs.diff.tar.gzip", + "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", + ], + }, + "size": {"type": "number"}, + "digest": {"type": "string"}, + }, + "required": ["mediaType", "size", "digest"], + }, + }, + }, + "required": ["schemaVersion", "mediaType", "config", "layers"], +} + +DOCKER_MANIFEST_V1_SCHEMA = { + "type": "object", + "properties": { + "signatures": { + "type": "array", + "items": { + "type": "object", + "properties": { + "protected": {"type": "string"}, + "header": { + "type": "object", + "properties": {"alg": {"type": "string"}, "jwk": {"type": "object"}}, + "required": ["alg", "jwk"], + }, + "signature": {"type": "string"}, + }, + "required": ["protected", "header", "signature"], + }, + }, + "tag": {"type": "string"}, + "name": {"type": "string"}, + "history": { + "type": "array", + "items": { + "type": "object", + "properties": {"v1Compatibility": {"type": "string"}}, + "required": ["v1Compatibility"], + }, + }, + "fsLayers": { + "type": "array", + "items": { + "type": "object", + "properties": {"blobSum": {"type": "string"}}, + "required": ["blobSum"], + }, + }, + }, + "required": ["tag", "name", "fsLayers", "history"], +} + +SIGNATURE_SCHEMA = { "title": "Atomic Container Signature", "description": "JSON Schema Validation for the Signature Payload", "type": "object", @@ -8,47 +221,31 @@ "critical": { "type": "object", "properties": { - "type": { - "type": "string", - "const": "atomic container signature" - }, + "type": {"type": "string", "const": "atomic container signature"}, "image": { "type": "object", - "properties": { - "docker-manifest-digest": { - "type": "string" - } - }, + "properties": {"docker-manifest-digest": {"type": "string"}}, "required": ["docker-manifest-digest"], - "additionalProperties": false + "additionalProperties": False, }, "identity": { "type": "object", - "properties": { - "docker-reference": { - "type": "string" - } - }, + "properties": {"docker-reference": {"type": "string"}}, "required": ["docker-reference"], - "additionalProperties": false - } + "additionalProperties": False, + }, }, "required": ["type", "image", "identity"], - "additionalProperties": false + "additionalProperties": False, }, "optional": { "type": "object", "properties": { - "creator": { - "type": "string" - }, - "timestamp": { - "type": "number", - "minimum": 0 - } - } - } + "creator": {"type": "string"}, + "timestamp": {"type": "number", "minimum": 0}, + }, + }, }, "required": ["critical", "optional"], - "additionalProperties": false -}""" + "additionalProperties": False, +} diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py index cc9215211..c1fd8f0ba 100644 --- a/pulp_container/app/registry_api.py +++ b/pulp_container/app/registry_api.py @@ -23,6 +23,7 @@ from django.shortcuts import get_object_or_404 from django.conf import settings +from jsonschema import validate, ValidationError as SchemaValidationError from pulpcore.plugin.models import Artifact, ContentArtifact, Task, UploadChunk from pulpcore.plugin.files import PulpTemporaryUploadedFile @@ -71,7 +72,11 @@ RegistryPermission, TokenPermission, ) -from pulp_container.app.utils import extract_data_from_signature, has_task_completed +from pulp_container.app.utils import ( + determine_schema, + extract_data_from_signature, + has_task_completed, +) from pulp_container.constants import ( EMPTY_BLOB, SIGNATURE_API_EXTENSION_VERSION, @@ -867,14 +872,6 @@ def put(self, request, path, pk=None): """ Responds with the actual manifest """ - # when a user uploads a manifest list with zero listed manifests (no blobs were uploaded - # before) and the specified repository has not been created yet, create the repository - # without raising an error - create_new_repo = request.content_type in ( - models.MEDIA_TYPE.MANIFEST_LIST, - models.MEDIA_TYPE.INDEX_OCI, - ) - _, repository = self.get_dr_push(request, path, create=create_new_repo) # iterate over all the layers and create chunk = request.META["wsgi.input"] artifact = self.receive_artifact(chunk) @@ -895,6 +892,20 @@ def put(self, request, path, pk=None): content_data = json.loads(raw_data) + try: + self.validate_manifest(content_data, request.content_type) + except SchemaValidationError: + raise ManifestInvalid(digest=manifest_digest) + + # when a user uploads a manifest list with zero listed manifests (no blobs were uploaded + # before) and the specified repository has not been created yet, create the repository + # without raising an error + create_new_repo = request.content_type in ( + models.MEDIA_TYPE.MANIFEST_LIST, + models.MEDIA_TYPE.INDEX_OCI, + ) + _, repository = self.get_dr_push(request, path, create=create_new_repo) + if request.content_type in ( models.MEDIA_TYPE.MANIFEST_LIST, models.MEDIA_TYPE.INDEX_OCI, @@ -1019,6 +1030,15 @@ def put(self, request, path, pk=None): if has_task_completed(dispatched_task): return ManifestResponse(manifest, path, request, status=201) + def validate_manifest(self, content_data, content_type): + """Validate JSON data (manifest) according to its declared content type (e.g., list).""" + try: + schema = determine_schema(content_type) + except ValueError: + raise ValidationError() + else: + validate(content_data, schema) + def _save_manifest(self, artifact, manifest_digest, content_type, config_blob=None): manifest = models.Manifest( digest=manifest_digest, diff --git a/pulp_container/app/tasks/sync_stages.py b/pulp_container/app/tasks/sync_stages.py index 55ebde225..0cd3b9f0d 100644 --- a/pulp_container/app/tasks/sync_stages.py +++ b/pulp_container/app/tasks/sync_stages.py @@ -6,6 +6,7 @@ import json import logging +from jsonschema import validate from urllib.parse import urljoin, urlparse, urlunparse from asgiref.sync import sync_to_async @@ -28,7 +29,11 @@ ManifestSignature, Tag, ) -from pulp_container.app.utils import extract_data_from_signature, urlpath_sanitize +from pulp_container.app.utils import ( + determine_schema, + extract_data_from_signature, + urlpath_sanitize, +) log = logging.getLogger(__name__) @@ -130,6 +135,16 @@ async def run(self): content_data = json.loads(raw_data) media_type = content_data.get("mediaType") + + try: + schema = determine_schema(media_type) + except ValueError: + raise RuntimeError( + f"Requested a validation schema for an unknown media type: '{media_type}'" + ) + else: + validate(content_data, schema) + if media_type in (MEDIA_TYPE.MANIFEST_LIST, MEDIA_TYPE.INDEX_OCI): list_dc = self.create_tagged_manifest_list( tag_name, saved_artifact, content_data diff --git a/pulp_container/app/utils.py b/pulp_container/app/utils.py index 2e95c7368..254a4da9c 100644 --- a/pulp_container/app/utils.py +++ b/pulp_container/app/utils.py @@ -8,10 +8,18 @@ from pulpcore.plugin.models import Task -from pulp_container.app.json_schemas import SIGNATURE_SCHEMA +from pulp_container.app.json_schemas import ( + OCI_INDEX_SCHEMA, + OCI_MANIFEST_SCHEMA, + DOCKER_MANIFEST_LIST_V2_SCHEMA, + DOCKER_MANIFEST_V2_SCHEMA, + DOCKER_MANIFEST_V1_SCHEMA, + SIGNATURE_SCHEMA, +) +from pulp_container.constants import MEDIA_TYPE -validator = Draft7Validator(json.loads(SIGNATURE_SCHEMA)) +validator = Draft7Validator(SIGNATURE_SCHEMA) log = logging.getLogger(__name__) @@ -122,3 +130,19 @@ def has_task_completed(dispatched_task): task.delete() raise Exception(str(error)) raise Throttled() + + +def determine_schema(content_type): + """Return a JSON schema based on the specified content type.""" + if content_type == MEDIA_TYPE.MANIFEST_V2: + return DOCKER_MANIFEST_V2_SCHEMA + elif content_type == MEDIA_TYPE.MANIFEST_OCI: + return OCI_MANIFEST_SCHEMA + elif content_type == MEDIA_TYPE.MANIFEST_LIST: + return DOCKER_MANIFEST_LIST_V2_SCHEMA + elif content_type == MEDIA_TYPE.INDEX_OCI: + return OCI_INDEX_SCHEMA + elif content_type in (MEDIA_TYPE.MANIFEST_V1, MEDIA_TYPE.MANIFEST_V1_SIGNED): + return DOCKER_MANIFEST_V1_SCHEMA + else: + raise ValueError() diff --git a/pulp_container/tests/unit/test_json_schemas.py b/pulp_container/tests/unit/test_json_schemas.py index e153c4d58..5e38e6d07 100644 --- a/pulp_container/tests/unit/test_json_schemas.py +++ b/pulp_container/tests/unit/test_json_schemas.py @@ -6,7 +6,7 @@ from pulp_container.app.json_schemas import SIGNATURE_SCHEMA -validator = Draft7Validator(json.loads(SIGNATURE_SCHEMA)) +validator = Draft7Validator(SIGNATURE_SCHEMA) class TestSignatureJsonSchema(TestCase):