refactor: python typing improvements (#2537)

A series of typing improvements I made while working on the cursor
changes. No logic or behavior changes have been made, so should perform
identically with the existing deployed code.

Changes:
- Add mypy-protobuf extension, which enables the `--mypy_out=` flag for
protoc. This generates a typing stub for the grpc protobuf output, which
can be used by different python type checking tools/extensions. (does
not have to be mypy)
- Generate and commit the pyi files, and update the readme / script to
help generate them in the future.
- Move helper functions in models.py to their own file
(model_helpers.py)
- Add explicit type definitions to models in `models.py`
- These types are not technically "correct", but are the types that will
get returned by datastore. This allows us to more easily label the
responses from ndb queries.
- Added an exclusion to the assignment type check of models.py to the
pyproject.toml for pyright (used by vscode and pylance). If you use a
different type checker, you might want to look into ignoring the errors
in models.py in your own config.

Sadly yield still causes problems typing wise, so turning on "strict"
typing is still not possible, details for anyone interested in why:

> Using the yield anywhere in a function turns the function into a
generator,
> The left hand side output of yield is received when the caller calls
send() on the returned generator.
> Python assumes that the sent type will always be the same throughout a
single function, so the send type is defined at the top of the function.
> ndb uses yield as its own custom async implementation, the left hand
side output of yield is the result of the future returned from the right
hand side of yield.
> This means the output of a yield call will depend on the right hand
side of the yield, which is impossible to represent in python's type
hinting system.

gcp/api/osv_service_v1_pb2.pyi

@@ -0,0 +1,368 @@
+"""
+@generated by mypy-protobuf.  Do not edit manually!
+isort:skip_file
+Copyright 2021 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import builtins
+import collections.abc
+import google.protobuf.descriptor
+import google.protobuf.internal.containers
+import google.protobuf.internal.enum_type_wrapper
+import google.protobuf.message
+import osv.vulnerability_pb2
+import sys
+import typing
+
+if sys.version_info >= (3, 10):
+    import typing as typing_extensions
+else:
+    import typing_extensions
+
+DESCRIPTOR: google.protobuf.descriptor.FileDescriptor
+
+@typing.final
+class VulnerabilityList(google.protobuf.message.Message):
+    """A list of Vulnerability entries."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    VULNS_FIELD_NUMBER: builtins.int
+    NEXT_PAGE_TOKEN_FIELD_NUMBER: builtins.int
+    next_page_token: builtins.str
+    @property
+    def vulns(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[osv.vulnerability_pb2.Vulnerability]: ...
+    def __init__(
+        self,
+        *,
+        vulns: collections.abc.Iterable[osv.vulnerability_pb2.Vulnerability] | None = ...,
+        next_page_token: builtins.str = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["next_page_token", b"next_page_token", "vulns", b"vulns"]) -> None: ...
+
+global___VulnerabilityList = VulnerabilityList
+
+@typing.final
+class BatchVulnerabilityList(google.protobuf.message.Message):
+    """Batched lists of Vulnerability entries."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    RESULTS_FIELD_NUMBER: builtins.int
+    @property
+    def results(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___VulnerabilityList]: ...
+    def __init__(
+        self,
+        *,
+        results: collections.abc.Iterable[global___VulnerabilityList] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["results", b"results"]) -> None: ...
+
+global___BatchVulnerabilityList = BatchVulnerabilityList
+
+@typing.final
+class Query(google.protobuf.message.Message):
+    """Query format."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    COMMIT_FIELD_NUMBER: builtins.int
+    VERSION_FIELD_NUMBER: builtins.int
+    PACKAGE_FIELD_NUMBER: builtins.int
+    PAGE_TOKEN_FIELD_NUMBER: builtins.int
+    commit: builtins.str
+    """The commit hash to query for. If specified, `version` should not be set."""
+    version: builtins.str
+    """The version string to query for. A fuzzy match is done against upstream
+    versions. If specified, `commit` should not be set.
+    """
+    page_token: builtins.str
+    @property
+    def package(self) -> osv.vulnerability_pb2.Package:
+        """The package to query against. When a `commit` hash is given, this is
+        optional.
+        """
+
+    def __init__(
+        self,
+        *,
+        commit: builtins.str = ...,
+        version: builtins.str = ...,
+        package: osv.vulnerability_pb2.Package | None = ...,
+        page_token: builtins.str = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing.Literal["commit", b"commit", "package", b"package", "param", b"param", "version", b"version"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["commit", b"commit", "package", b"package", "page_token", b"page_token", "param", b"param", "version", b"version"]) -> None: ...
+    def WhichOneof(self, oneof_group: typing.Literal["param", b"param"]) -> typing.Literal["commit", "version"] | None: ...
+
+global___Query = Query
+
+@typing.final
+class BatchQuery(google.protobuf.message.Message):
+    """Batch query format."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    QUERIES_FIELD_NUMBER: builtins.int
+    @property
+    def queries(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Query]:
+        """The queries that form this batch query."""
+
+    def __init__(
+        self,
+        *,
+        queries: collections.abc.Iterable[global___Query] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["queries", b"queries"]) -> None: ...
+
+global___BatchQuery = BatchQuery
+
+@typing.final
+class GetVulnByIdParameters(google.protobuf.message.Message):
+    """Parameters for GetBugById."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    ID_FIELD_NUMBER: builtins.int
+    id: builtins.str
+    def __init__(
+        self,
+        *,
+        id: builtins.str = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["id", b"id"]) -> None: ...
+
+global___GetVulnByIdParameters = GetVulnByIdParameters
+
+@typing.final
+class QueryAffectedParameters(google.protobuf.message.Message):
+    """Parameters for QueryAffected."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    QUERY_FIELD_NUMBER: builtins.int
+    @property
+    def query(self) -> global___Query: ...
+    def __init__(
+        self,
+        *,
+        query: global___Query | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing.Literal["query", b"query"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["query", b"query"]) -> None: ...
+
+global___QueryAffectedParameters = QueryAffectedParameters
+
+@typing.final
+class QueryAffectedBatchParameters(google.protobuf.message.Message):
+    """Parameters for QueryAffectedBatch."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    QUERY_FIELD_NUMBER: builtins.int
+    @property
+    def query(self) -> global___BatchQuery: ...
+    def __init__(
+        self,
+        *,
+        query: global___BatchQuery | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing.Literal["query", b"query"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["query", b"query"]) -> None: ...
+
+global___QueryAffectedBatchParameters = QueryAffectedBatchParameters
+
+@typing.final
+class DetermineVersionParameters(google.protobuf.message.Message):
+    """Parameters for DetermineVersion."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    QUERY_FIELD_NUMBER: builtins.int
+    @property
+    def query(self) -> global___VersionQuery: ...
+    def __init__(
+        self,
+        *,
+        query: global___VersionQuery | None = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing.Literal["query", b"query"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["query", b"query"]) -> None: ...
+
+global___DetermineVersionParameters = DetermineVersionParameters
+
+@typing.final
+class VersionQuery(google.protobuf.message.Message):
+    """The version query."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    NAME_FIELD_NUMBER: builtins.int
+    FILE_HASHES_FIELD_NUMBER: builtins.int
+    name: builtins.str
+    """The name of the dependency. Can be empty."""
+    @property
+    def file_hashes(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FileHash]: ...
+    def __init__(
+        self,
+        *,
+        name: builtins.str = ...,
+        file_hashes: collections.abc.Iterable[global___FileHash] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["file_hashes", b"file_hashes", "name", b"name"]) -> None: ...
+
+global___VersionQuery = VersionQuery
+
+@typing.final
+class FileHash(google.protobuf.message.Message):
+    """Information about the files in the repository
+    to identify the version.
+    """
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    class _HashType:
+        ValueType = typing.NewType("ValueType", builtins.int)
+        V: typing_extensions.TypeAlias = ValueType
+
+    class _HashTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[FileHash._HashType.ValueType], builtins.type):
+        DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+        MD5: FileHash._HashType.ValueType  # 0
+
+    class HashType(_HashType, metaclass=_HashTypeEnumTypeWrapper): ...
+    MD5: FileHash.HashType.ValueType  # 0
+
+    FILE_PATH_FIELD_NUMBER: builtins.int
+    HASH_TYPE_FIELD_NUMBER: builtins.int
+    HASH_FIELD_NUMBER: builtins.int
+    file_path: builtins.str
+    """The file path inside the repository, relative to the repository root."""
+    hash_type: global___FileHash.HashType.ValueType
+    hash: builtins.bytes
+    def __init__(
+        self,
+        *,
+        file_path: builtins.str = ...,
+        hash_type: global___FileHash.HashType.ValueType = ...,
+        hash: builtins.bytes = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["file_path", b"file_path", "hash", b"hash", "hash_type", b"hash_type"]) -> None: ...
+
+global___FileHash = FileHash
+
+@typing.final
+class VersionMatchList(google.protobuf.message.Message):
+    """Result of DetmineVersion."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    MATCHES_FIELD_NUMBER: builtins.int
+    @property
+    def matches(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___VersionMatch]: ...
+    def __init__(
+        self,
+        *,
+        matches: collections.abc.Iterable[global___VersionMatch] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["matches", b"matches"]) -> None: ...
+
+global___VersionMatchList = VersionMatchList
+
+@typing.final
+class VersionMatch(google.protobuf.message.Message):
+    """Match information for the provided VersionQuery."""
+
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    SCORE_FIELD_NUMBER: builtins.int
+    REPO_INFO_FIELD_NUMBER: builtins.int
+    OSV_IDENTIFIER_FIELD_NUMBER: builtins.int
+    CPE23_FIELD_NUMBER: builtins.int
+    MINIMUM_FILE_MATCHES_FIELD_NUMBER: builtins.int
+    ESTIMATED_DIFF_FILES_FIELD_NUMBER: builtins.int
+    score: builtins.float
+    """Score in the interval (0.0, 1.0] with 1.0 being a perfect match."""
+    cpe23: builtins.str
+    """CPE 2.3."""
+    minimum_file_matches: builtins.int
+    """Definite number of files that have matched."""
+    estimated_diff_files: builtins.int
+    """Estimated number of files that are different."""
+    @property
+    def repo_info(self) -> global___VersionRepositoryInformation:
+        """Information about the upstream repository."""
+
+    @property
+    def osv_identifier(self) -> osv.vulnerability_pb2.Package:
+        """The OSV identifier."""
+
+    def __init__(
+        self,
+        *,
+        score: builtins.float = ...,
+        repo_info: global___VersionRepositoryInformation | None = ...,
+        osv_identifier: osv.vulnerability_pb2.Package | None = ...,
+        cpe23: builtins.str = ...,
+        minimum_file_matches: builtins.int = ...,
+        estimated_diff_files: builtins.int = ...,
+    ) -> None: ...
+    def HasField(self, field_name: typing.Literal["osv_identifier", b"osv_identifier", "repo_info", b"repo_info"]) -> builtins.bool: ...
+    def ClearField(self, field_name: typing.Literal["cpe23", b"cpe23", "estimated_diff_files", b"estimated_diff_files", "minimum_file_matches", b"minimum_file_matches", "osv_identifier", b"osv_identifier", "repo_info", b"repo_info", "score", b"score"]) -> None: ...
+
+global___VersionMatch = VersionMatch
+
+@typing.final
+class VersionRepositoryInformation(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    class _RepoType:
+        ValueType = typing.NewType("ValueType", builtins.int)
+        V: typing_extensions.TypeAlias = ValueType
+
+    class _RepoTypeEnumTypeWrapper(google.protobuf.internal.enum_type_wrapper._EnumTypeWrapper[VersionRepositoryInformation._RepoType.ValueType], builtins.type):
+        DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
+        UNSPECIFIED: VersionRepositoryInformation._RepoType.ValueType  # 0
+        GIT: VersionRepositoryInformation._RepoType.ValueType  # 1
+
+    class RepoType(_RepoType, metaclass=_RepoTypeEnumTypeWrapper): ...
+    UNSPECIFIED: VersionRepositoryInformation.RepoType.ValueType  # 0
+    GIT: VersionRepositoryInformation.RepoType.ValueType  # 1
+
+    TYPE_FIELD_NUMBER: builtins.int
+    ADDRESS_FIELD_NUMBER: builtins.int
+    TAG_FIELD_NUMBER: builtins.int
+    VERSION_FIELD_NUMBER: builtins.int
+    COMMIT_FIELD_NUMBER: builtins.int
+    type: global___VersionRepositoryInformation.RepoType.ValueType
+    address: builtins.str
+    """Source address of the repository."""
+    tag: builtins.str
+    """Commit tag"""
+    version: builtins.str
+    """Parsed version from commit tag"""
+    commit: builtins.str
+    """Commit hash hex."""
+    def __init__(
+        self,
+        *,
+        type: global___VersionRepositoryInformation.RepoType.ValueType = ...,
+        address: builtins.str = ...,
+        tag: builtins.str = ...,
+        version: builtins.str = ...,
+        commit: builtins.str = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing.Literal["address", b"address", "commit", b"commit", "tag", b"tag", "type", b"type", "version", b"version"]) -> None: ...
+
+global___VersionRepositoryInformation = VersionRepositoryInformation