refactor addon related serializers & fields; add source field

docs/topics/api/addons.rst

@@ -459,6 +459,7 @@ This endpoint allows you to fetch a single version belonging to a specific add-o
     :>json object|null release_notes: The release notes for this version (See :ref:`translated fields <api-overview-translations>`).
     :>json string reviewed: The date the version was reviewed at.
     :>json boolean is_strict_compatibility_enabled: Whether or not this version has `strictCompatibility <https://developer.mozilla.org/en-US/Add-ons/Install_Manifests#strictCompatibility>`_. set.
+    :>json string|null source: The (absolute) URL to download the submitted source for this version. This field is only present for authenticated users, for their own add-ons.
     :>json string version: The version number string for the version.
 
 
@@ -493,6 +494,7 @@ This endpoint allows a submission of an upload to an existing add-on to create a
     :<json object|null custom_license.text: The text of the license (See :ref:`translated fields <api-overview-translations>`). Custom licenses are not supported for themes.
     :<json object|null release_notes: The release notes for this version (See :ref:`translated fields <api-overview-translations>`).
     :<json string upload: The uuid for the xpi upload to create this version with.
+    :<json string|null: source: The submitted source for this version. As JSON this field can only be set to null, to clear it - see :ref:`uploading source <version-sources>` to set/update the source file.
 
 
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -547,6 +549,32 @@ Shorthand, for when you only want to define compatible apps, but use the min/max
     }
 
 
+~~~~~~~~~~~~~~~
+Version Sources
+~~~~~~~~~~~~~~~
+
+.. _version-sources:
+
+Version source files cannot be uploaded as JSON - the request must be sent as form-data instead.
+Other fields can be set/updated at the same time as ``source`` if desired.
+
+.. http:post:: /api/v5/addons/addon/(int:addon_id|string:addon_slug|string:addon_guid)/versions/
+
+    .. _version-sources-request-create:
+
+    :form source: The add-on file being uploaded.
+    :form compatibility: See :ref:`create <version-create-request>` for details.
+    :form upload: The uuid for the xpi upload to create this version with.
+    :reqheader Content-Type: multipart/form-data
+
+
+.. http:patch:: /api/v5/addons/addon/(int:addon_id|string:addon_slug|string:addon_guid)/versions/(int:id)/
+
+    .. _version-sources-request-edit:
+
+    :form source: The add-on file being uploaded.
+    :reqheader Content-Type: multipart/form-data
+
 ------------
 Version Edit
 ------------
@@ -574,6 +602,7 @@ This endpoint allows the metadata for an existing version to be edited.
     :<json object|null custom_license.name: The name of the license (See :ref:`translated fields <api-overview-translations>`). Custom licenses are not supported for themes.
     :<json object|null custom_license.text: The text of the license (See :ref:`translated fields <api-overview-translations>`). Custom licenses are not supported for themes.
     :<json object|null release_notes: The release notes for this version (See :ref:`translated fields <api-overview-translations>`).
+    :<json string|null: source: The submitted source for this version. As JSON this field can only be set to null, to clear it - see :ref:`uploading source <version-sources>` to set/update the source file.
 
 
 -------------

docs/topics/api/overview.rst

@@ -435,6 +435,7 @@ These are `v5` specific changes - `v4` changes apply also.
 * 2021-12-09: enabled setting ``tags`` via addon submission and edit apis. https://github.com/mozilla/addons-server/issues/18268
 * 2021-12-09: changed ``license`` in version create/update endpoints to accept a license slug rather than numeric ID, and documented supported licenses. https://github.com/mozilla/addons-server/issues/18361
 * 2022-01-27: added ``ERROR_AUTHENTICATION_EXPIRED`` error code for authentication failures. https://github.com/mozilla/addons-server/issues/18669
+* 2022-02-03: added ``source`` to version detail responses, for developer's own add-ons. https://github.com/mozilla/addons-server/issues/9913
 
 .. _`#11380`: https://github.com/mozilla/addons-server/issues/11380/
 .. _`#11379`: https://github.com/mozilla/addons-server/issues/11379/

src/olympia/addons/fields.py

@@ -0,0 +1,267 @@
+import os
+import tarfile
+import zipfile
+
+from urllib.parse import urlsplit, urlunsplit
+
+from django.http.request import QueryDict
+from django.urls import reverse
+
+from rest_framework import fields, exceptions, serializers
+
+from olympia import amo
+from olympia.amo.utils import sorted_groupby
+from olympia.amo.templatetags.jinja_helpers import absolutify
+from olympia.api.fields import (
+    ESTranslationSerializerField,
+    GetTextTranslationSerializerField,
+    OutgoingURLField,
+    TranslationSerializerField,
+)
+from olympia.applications.models import AppVersion
+from olympia.constants.applications import APPS
+from olympia.constants.categories import CATEGORIES
+from olympia.constants.licenses import LICENSES_BY_SLUG
+from olympia.files.utils import SafeZip, archive_member_validator
+from olympia.versions.models import (
+    ApplicationsVersions,
+    License,
+    VALID_SOURCE_EXTENSIONS,
+)
+
+
+class CategoriesSerializerField(serializers.Field):
+    def to_internal_value(self, data):
+        try:
+            categories = []
+            for app_name, category_names in data.items():
+                if len(category_names) > amo.MAX_CATEGORIES:
+                    raise exceptions.ValidationError(
+                        'Maximum number of categories per application '
+                        f'({amo.MAX_CATEGORIES}) exceeded'
+                    )
+                if len(category_names) > 1 and 'other' in category_names:
+                    raise exceptions.ValidationError(
+                        'The "other" category cannot be combined with another category'
+                    )
+                app_cats = CATEGORIES[APPS[app_name].id]
+                # We don't know the addon_type at this point, so try them all and we'll
+                # drop anything that's wrong later in AddonSerializer.validate
+                all_cat_slugs = set()
+                for type_cats in app_cats.values():
+                    categories.extend(
+                        type_cats[name] for name in category_names if name in type_cats
+                    )
+                    all_cat_slugs.update(type_cats.keys())
+                # Now double-check all the category names were found
+                if not all_cat_slugs.issuperset(category_names):
+                    raise exceptions.ValidationError('Invalid category name.')
+            return categories
+        except KeyError:
+            raise exceptions.ValidationError('Invalid app name.')
+
+    def to_representation(self, value):
+        grouped = sorted_groupby(
+            sorted(value),
+            key=lambda x: getattr(amo.APP_IDS.get(x.application), 'short', ''),
+        )
+        return {
+            app_name: [cat.slug for cat in categories]
+            for app_name, categories in grouped
+        }
+
+
+class ContributionSerializerField(OutgoingURLField):
+    def to_representation(self, value):
+        if not value:
+            # don't add anything when it's not set.
+            return value
+        parts = urlsplit(value)
+        query = QueryDict(parts.query, mutable=True)
+        query.update(amo.CONTRIBUTE_UTM_PARAMS)
+        return super().to_representation(
+            urlunsplit(
+                (
+                    parts.scheme,
+                    parts.netloc,
+                    parts.path,
+                    query.urlencode(),
+                    parts.fragment,
+                )
+            )
+        )
+
+
+class LicenseNameSerializerField(serializers.Field):
+    """Field to handle license name translations.
+
+    Builtin licenses, for better or worse, don't necessarily have their name
+    translated in the database like custom licenses. Instead, the string is in
+    this repos, and translated using gettext. This field deals with that
+    difference, delegating the rendering to TranslationSerializerField or
+    GetTextTranslationSerializerField depending on what the license instance
+    is.
+    """
+
+    builtin_translation_field_class = GetTextTranslationSerializerField
+    custom_translation_field_class = TranslationSerializerField
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.builtin_translation_field = self.builtin_translation_field_class()
+        self.custom_translation_field = self.custom_translation_field_class()
+
+    def bind(self, field_name, parent):
+        super().bind(field_name, parent)
+        self.builtin_translation_field.bind(field_name, parent)
+        self.custom_translation_field.bind(field_name, parent)
+
+    def get_attribute(self, obj):
+        if obj._constant:
+            return self.builtin_translation_field.get_attribute(obj._constant)
+        else:
+            return self.custom_translation_field.get_attribute(obj)
+
+    def to_representation(self, obj):
+        # Like TranslationSerializerField, the bulk of the logic is in
+        # get_attribute(), we just have to return the data at this point.
+        return obj
+
+    def run_validation(self, data=fields.empty):
+        return self.custom_translation_field.run_validation(data)
+
+    def to_internal_value(self, value):
+        return self.custom_translation_field.to_internal_value(value)
+
+
+class ESLicenseNameSerializerField(LicenseNameSerializerField):
+    """Like LicenseNameSerializerField, but uses the data from ES to avoid
+    a database query for custom licenses.
+
+    BaseESSerializer automatically changes
+    TranslationSerializerField to ESTranslationSerializerField for all base
+    fields on the serializer, but License name has its own special field to
+    handle builtin licences so it's done separately."""
+
+    custom_translation_field_class = ESTranslationSerializerField
+
+    def attach_translations(self, obj, data, field_name):
+        return self.custom_translation_field.attach_translations(obj, data, field_name)
+
+
+class LicenseSlugSerializerField(serializers.SlugRelatedField):
+    def __init__(self, **kwargs):
+        super().__init__(
+            slug_field='builtin',
+            queryset=License.objects.exclude(builtin=License.OTHER),
+            **kwargs,
+        )
+
+    def to_internal_value(self, data):
+        license_ = LICENSES_BY_SLUG.get(data)
+        if not license_:
+            self.fail('invalid')
+        return super().to_internal_value(license_.builtin)
+
+
+class SourceFileField(serializers.FileField):
+    def to_internal_value(self, data):
+        data = super().to_internal_value(data)
+
+        # Ensure the file type is one we support.
+        if not data.name.endswith(VALID_SOURCE_EXTENSIONS):
+            error_msg = (
+                'Unsupported file type, please upload an archive file ({extensions}).'
+            )
+            raise exceptions.ValidationError(
+                error_msg.format(extensions=(', '.join(VALID_SOURCE_EXTENSIONS)))
+            )
+
+        # Check inside to see if the file extension matches the content.
+        try:
+            _, ext = os.path.splitext(data.name)
+            if ext == '.zip':
+                # testzip() returns None if there are no broken CRCs.
+                if SafeZip(data).zip_file.testzip() is not None:
+                    raise zipfile.BadZipFile()
+            else:
+                # For tar files we need to do a little more work.
+                mode = 'r:bz2' if ext == '.bz2' else 'r:gz'
+                with tarfile.open(mode=mode, fileobj=data) as archive:
+                    for member in archive.getmembers():
+                        archive_member_validator(archive, member)
+        except (zipfile.BadZipFile, tarfile.ReadError, OSError, EOFError):
+            raise exceptions.ValidationError('Invalid or broken archive.')
+
+        return data
+
+    def to_representation(self, value):
+        if not value:
+            return None
+        else:
+            return absolutify(reverse('downloads.source', args=(self.parent.id,)))
+
+
+class VersionCompatabilityField(serializers.Field):
+    def to_internal_value(self, data):
+        """Note: this returns unsaved and incomplete ApplicationsVersions objects that
+        need to have version set, and may have missing min or max AppVersion instances
+        for new Version instances. (As intended - we want to be able to partially
+        specify min or max and have the manifest or defaults be instead used).
+        """
+        try:
+            if isinstance(data, list):
+                # if it's a list of apps, normalize into a dict first
+                data = {key: {} for key in data}
+            if isinstance(data, dict):
+                version = self.parent.instance
+                existing = version.compatible_apps if version else {}
+                qs = AppVersion.objects
+                internal = {}
+                for app_name, min_max in data.items():
+                    app = amo.APPS[app_name]
+                    apps_versions = existing.get(
+                        app, ApplicationsVersions(application=app.id)
+                    )
+
+                    app_qs = qs.filter(application=app.id)
+                    if 'max' in min_max:
+                        apps_versions.max = app_qs.get(version=min_max['max'])
+                    elif version:
+                        apps_versions.max = app_qs.get(
+                            version=amo.DEFAULT_WEBEXT_MAX_VERSION
+                        )
+
+                    app_qs = app_qs.exclude(version='*')
+                    if 'min' in min_max:
+                        apps_versions.min = app_qs.get(version=min_max['min'])
+                    elif version:
+                        apps_versions.min = app_qs.get(
+                            version=amo.DEFAULT_WEBEXT_MIN_VERSIONS[app]
+                        )
+
+                    internal[app] = apps_versions
+                return internal
+            else:
+                # if it's neither it's not a valid input
+                raise exceptions.ValidationError('Invalid value')
+        except KeyError:
+            raise exceptions.ValidationError('Invalid app specified')
+        except AppVersion.DoesNotExist:
+            raise exceptions.ValidationError('Unknown app version specified')
+
+    def to_representation(self, value):
+        return {
+            app.short: (
+                {
+                    'min': compat.min.version,
+                    'max': compat.max.version,
+                }
+                if compat
+                else {
+                    'min': amo.D2C_MIN_VERSIONS.get(app.id, '1.0'),
+                    'max': amo.FAKE_MAX_VERSION,
+                }
+            )
+            for app, compat in value.items()
+        }