diff --git a/packages/google-cloud-dlp/.github/.OwlBot.lock.yaml b/packages/google-cloud-dlp/.github/.OwlBot.lock.yaml index 2185b591844c..d6ac3e7a5bb8 100644 --- a/packages/google-cloud-dlp/.github/.OwlBot.lock.yaml +++ b/packages/google-cloud-dlp/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:65e656411895bff71cffcae97246966460160028f253c2e45b7a25d805a5b142 -# created: 2022-06-12T13:11:45.905884945Z + digest: sha256:e7bb19d47c13839fe8c147e50e02e8b6cf5da8edd1af8b82208cd6f66cc2829c +# created: 2022-07-05T18:31:20.838186805Z diff --git a/packages/google-cloud-dlp/.github/workflows/unittest.yml b/packages/google-cloud-dlp/.github/workflows/unittest.yml index e5be6edbd54d..5531b0141297 100644 --- a/packages/google-cloud-dlp/.github/workflows/unittest.yml +++ b/packages/google-cloud-dlp/.github/workflows/unittest.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python: ['3.6', '3.7', '3.8', '3.9', '3.10'] + python: ['3.7', '3.8', '3.9', '3.10'] steps: - name: Checkout uses: actions/checkout@v3 diff --git a/packages/google-cloud-dlp/.kokoro/samples/python3.6/common.cfg b/packages/google-cloud-dlp/.kokoro/samples/python3.6/common.cfg deleted file mode 100644 index 9d9252cd5c7a..000000000000 --- a/packages/google-cloud-dlp/.kokoro/samples/python3.6/common.cfg +++ /dev/null @@ -1,40 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -# Build logs will be here -action { - define_artifacts { - regex: "**/*sponge_log.xml" - } -} - -# Specify which tests to run -env_vars: { - key: "RUN_TESTS_SESSION" - value: "py-3.6" -} - -# Declare build specific Cloud project. -env_vars: { - key: "BUILD_SPECIFIC_GCLOUD_PROJECT" - value: "python-docs-samples-tests-py36" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-dlp/.kokoro/test-samples.sh" -} - -# Configure the docker image for kokoro-trampoline. -env_vars: { - key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/python-samples-testing-docker" -} - -# Download secrets for samples -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/python-docs-samples" - -# Download trampoline resources. -gfile_resources: "/bigstore/cloud-devrel-kokoro-resources/trampoline" - -# Use the trampoline script to run in docker. -build_file: "python-dlp/.kokoro/trampoline_v2.sh" \ No newline at end of file diff --git a/packages/google-cloud-dlp/.kokoro/samples/python3.6/continuous.cfg b/packages/google-cloud-dlp/.kokoro/samples/python3.6/continuous.cfg deleted file mode 100644 index 7218af1499e5..000000000000 --- a/packages/google-cloud-dlp/.kokoro/samples/python3.6/continuous.cfg +++ /dev/null @@ -1,7 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - diff --git a/packages/google-cloud-dlp/.kokoro/samples/python3.6/periodic-head.cfg b/packages/google-cloud-dlp/.kokoro/samples/python3.6/periodic-head.cfg deleted file mode 100644 index a5428fdc16e1..000000000000 --- a/packages/google-cloud-dlp/.kokoro/samples/python3.6/periodic-head.cfg +++ /dev/null @@ -1,11 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} - -env_vars: { - key: "TRAMPOLINE_BUILD_FILE" - value: "github/python-dlp/.kokoro/test-samples-against-head.sh" -} diff --git a/packages/google-cloud-dlp/.kokoro/samples/python3.6/periodic.cfg b/packages/google-cloud-dlp/.kokoro/samples/python3.6/periodic.cfg deleted file mode 100644 index 71cd1e597e38..000000000000 --- a/packages/google-cloud-dlp/.kokoro/samples/python3.6/periodic.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "False" -} diff --git a/packages/google-cloud-dlp/.kokoro/samples/python3.6/presubmit.cfg b/packages/google-cloud-dlp/.kokoro/samples/python3.6/presubmit.cfg deleted file mode 100644 index a1c8d9759c88..000000000000 --- a/packages/google-cloud-dlp/.kokoro/samples/python3.6/presubmit.cfg +++ /dev/null @@ -1,6 +0,0 @@ -# Format: //devtools/kokoro/config/proto/build.proto - -env_vars: { - key: "INSTALL_LIBRARY_FROM_SOURCE" - value: "True" -} \ No newline at end of file diff --git a/packages/google-cloud-dlp/.kokoro/test-samples-impl.sh b/packages/google-cloud-dlp/.kokoro/test-samples-impl.sh index 8a324c9c7bc6..2c6500cae0b9 100755 --- a/packages/google-cloud-dlp/.kokoro/test-samples-impl.sh +++ b/packages/google-cloud-dlp/.kokoro/test-samples-impl.sh @@ -33,7 +33,7 @@ export PYTHONUNBUFFERED=1 env | grep KOKORO # Install nox -python3.6 -m pip install --upgrade --quiet nox +python3.9 -m pip install --upgrade --quiet nox # Use secrets acessor service account to get secrets if [[ -f "${KOKORO_GFILE_DIR}/secrets_viewer_service_account.json" ]]; then @@ -76,7 +76,7 @@ for file in samples/**/requirements.txt; do echo "------------------------------------------------------------" # Use nox to execute the tests for the project. - python3.6 -m nox -s "$RUN_TESTS_SESSION" + python3.9 -m nox -s "$RUN_TESTS_SESSION" EXIT=$? # If this is a periodic build, send the test log to the FlakyBot. diff --git a/packages/google-cloud-dlp/.repo-metadata.json b/packages/google-cloud-dlp/.repo-metadata.json index 14dd992e5813..2ab863b62ed2 100644 --- a/packages/google-cloud-dlp/.repo-metadata.json +++ b/packages/google-cloud-dlp/.repo-metadata.json @@ -13,5 +13,6 @@ "requires_billing": true, "default_version": "v2", "codeowner_team": "", - "api_shortname": "dlp" + "api_shortname": "dlp", + "api_description": "provides programmatic access to a powerful detection engine for personally identifiable information and other privacy-sensitive data in unstructured data streams, like text blocks and images." } diff --git a/packages/google-cloud-dlp/CONTRIBUTING.rst b/packages/google-cloud-dlp/CONTRIBUTING.rst index 9ac08506fd03..4e04fc24be6b 100644 --- a/packages/google-cloud-dlp/CONTRIBUTING.rst +++ b/packages/google-cloud-dlp/CONTRIBUTING.rst @@ -22,7 +22,7 @@ In order to add a feature: documentation. - The feature must work fully on the following CPython versions: - 3.6, 3.7, 3.8, 3.9 and 3.10 on both UNIX and Windows. + 3.7, 3.8, 3.9 and 3.10 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -221,13 +221,11 @@ Supported Python Versions We support: -- `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ - `Python 3.9`_ - `Python 3.10`_ -.. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ .. _Python 3.9: https://docs.python.org/3.9/ @@ -239,7 +237,7 @@ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-dlp/blob/main/noxfile.py -We also explicitly decided to support Python 3 beginning with version 3.6. +We also explicitly decided to support Python 3 beginning with version 3.7. Reasons for this include: - Encouraging use of newest versions of Python 3 diff --git a/packages/google-cloud-dlp/README.rst b/packages/google-cloud-dlp/README.rst index c5a6f1029382..54b01edf76f7 100644 --- a/packages/google-cloud-dlp/README.rst +++ b/packages/google-cloud-dlp/README.rst @@ -1,24 +1,22 @@ -Python Client for Cloud Data Loss Prevention (DLP) API -====================================================== +Python Client for Cloud Data Loss Prevention API +================================================ -|GA| |pypi| |versions| +|stable| |pypi| |versions| -`Cloud Data Loss Prevention (DLP) API`_: Provides methods for detection, risk analysis, and de-identification of -privacy-sensitive fragments in text, images, and Google Cloud Platform -storage repositories. +`Cloud Data Loss Prevention API`_: provides programmatic access to a powerful detection engine for personally identifiable information and other privacy-sensitive data in unstructured data streams, like text blocks and images. - `Client Library Documentation`_ - `Product Documentation`_ -.. |GA| image:: https://img.shields.io/badge/support-GA-gold.svg - :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#general-availability +.. |stable| image:: https://img.shields.io/badge/support-stable-gold.svg + :target: https://github.com/googleapis/google-cloud-python/blob/main/README.rst#stability-levels .. |pypi| image:: https://img.shields.io/pypi/v/google-cloud-dlp.svg :target: https://pypi.org/project/google-cloud-dlp/ .. |versions| image:: https://img.shields.io/pypi/pyversions/google-cloud-dlp.svg :target: https://pypi.org/project/google-cloud-dlp/ -.. _Cloud Data Loss Prevention (DLP) API: https://cloud.google.com/dlp +.. _Cloud Data Loss Prevention API: https://cloud.google.com/dlp/docs/ .. _Client Library Documentation: https://cloud.google.com/python/docs/reference/dlp/latest -.. _Product Documentation: https://cloud.google.com/dlp +.. _Product Documentation: https://cloud.google.com/dlp/docs/ Quick Start ----------- @@ -27,12 +25,12 @@ In order to use this library, you first need to go through the following steps: 1. `Select or create a Cloud Platform project.`_ 2. `Enable billing for your project.`_ -3. `Enable the Cloud Data Loss Prevention (DLP) API.`_ +3. `Enable the Cloud Data Loss Prevention API.`_ 4. `Setup Authentication.`_ .. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project .. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project -.. _Enable the Cloud Data Loss Prevention (DLP) API.: https://cloud.google.com/dlp +.. _Enable the Cloud Data Loss Prevention API.: https://cloud.google.com/dlp/docs/ .. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html Installation @@ -49,15 +47,25 @@ dependencies. .. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ +Code samples and snippets +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Code samples and snippets live in the `samples/` folder. + + Supported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^ -Python >= 3.6 +Our client libraries are compatible with all current [active](https://devguide.python.org/devcycle/#in-development-main-branch) and [maintenance](https://devguide.python.org/devcycle/#maintenance-branches) versions of +Python. -Deprecated Python Versions -^^^^^^^^^^^^^^^^^^^^^^^^^^ -Python == 2.7. +Python >= 3.7 -The last version of this library compatible with Python 2.7 is google-cloud-dlp==1.1.0. +Unsupported Python Versions +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Python <= 3.6 + +If you are using an [end-of-life](https://devguide.python.org/devcycle/#end-of-life-branches) +version of Python, we recommend that you update as soon as possible to an actively supported version. Mac/Linux @@ -81,33 +89,15 @@ Windows \Scripts\activate \Scripts\pip.exe install google-cloud-dlp -Preview -~~~~~~~ - -DlpServiceClient -^^^^^^^^^^^^^^^^ - -.. code:: py - - from google.cloud import dlp_v2 - - client = dlp_v2.DlpServiceClient() - - name = 'EMAIL_ADDRESS' - info_types_element = {'name': name} - info_types = [info_types_element] - inspect_config = {'info_types': info_types} - type_ = 'text/plain' - value = 'My email is example@example.com.' - items_element = {'type': type_, 'value': value} - items = [items_element] - - response = client.inspect_content(inspect_config, items) - Next Steps ~~~~~~~~~~ -- Read the `Client Library Documentation`_ for Cloud Data Loss Prevention - (DLP) API to see other available methods on the client. -- Read the `Product documentation`_ to - learn more about the product and see How-to Guides. +- Read the `Client Library Documentation`_ for Cloud Data Loss Prevention API + to see other available methods on the client. +- Read the `Cloud Data Loss Prevention API Product documentation`_ to learn + more about the product and see How-to Guides. +- View this `README`_ to see the full list of Cloud + APIs that we cover. + +.. _Cloud Data Loss Prevention API Product documentation: https://cloud.google.com/dlp/docs/ +.. _README: https://github.com/googleapis/google-cloud-python/blob/main/README.rst diff --git a/packages/google-cloud-dlp/noxfile.py b/packages/google-cloud-dlp/noxfile.py index b99ccd3f35f8..94b2f9c20760 100644 --- a/packages/google-cloud-dlp/noxfile.py +++ b/packages/google-cloud-dlp/noxfile.py @@ -20,6 +20,7 @@ import os import pathlib +import re import shutil import warnings @@ -31,7 +32,7 @@ DEFAULT_PYTHON_VERSION = "3.8" -UNIT_TEST_PYTHON_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] UNIT_TEST_STANDARD_DEPENDENCIES = [ "mock", "asyncmock", @@ -331,28 +332,15 @@ def docfx(session): def prerelease_deps(session): """Run all tests with prerelease versions of dependencies installed.""" - prerel_deps = [ - "protobuf", - "googleapis-common-protos", - "google-auth", - "grpcio", - "grpcio-status", - "google-api-core", - "proto-plus", - # dependencies of google-auth - "cryptography", - "pyasn1", - ] - - for dep in prerel_deps: - session.install("--pre", "--no-deps", "--upgrade", dep) - - # Remaining dependencies - other_deps = ["requests"] - session.install(*other_deps) - + # Install all dependencies + session.install("-e", ".[all, tests, tracing]") session.install(*UNIT_TEST_STANDARD_DEPENDENCIES) - session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES) + system_deps_all = ( + SYSTEM_TEST_STANDARD_DEPENDENCIES + + SYSTEM_TEST_EXTERNAL_DEPENDENCIES + + SYSTEM_TEST_EXTRAS + ) + session.install(*system_deps_all) # Because we test minimum dependency versions on the minimum Python # version, the first version we test with in the unit tests sessions has a @@ -366,19 +354,44 @@ def prerelease_deps(session): constraints_text = constraints_file.read() # Ignore leading whitespace and comment lines. - deps = [ + constraints_deps = [ match.group(1) for match in re.finditer( r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE ) ] - # Don't overwrite prerelease packages. - deps = [dep for dep in deps if dep not in prerel_deps] - # We use --no-deps to ensure that pre-release versions aren't overwritten - # by the version ranges in setup.py. - session.install(*deps) - session.install("--no-deps", "-e", ".[all]") + session.install(*constraints_deps) + + if os.path.exists("samples/snippets/requirements.txt"): + session.install("-r", "samples/snippets/requirements.txt") + + if os.path.exists("samples/snippets/requirements-test.txt"): + session.install("-r", "samples/snippets/requirements-test.txt") + + prerel_deps = [ + "protobuf", + # dependency of grpc + "six", + "googleapis-common-protos", + "grpcio", + "grpcio-status", + "google-api-core", + "proto-plus", + "google-cloud-testutils", + # dependencies of google-cloud-testutils" + "click", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + "google-auth", + ] + session.install(*other_deps) # Print out prerelease package versions session.run( @@ -387,5 +400,16 @@ def prerelease_deps(session): session.run("python", "-c", "import grpc; print(grpc.__version__)") session.run("py.test", "tests/unit") - session.run("py.test", "tests/system") - session.run("py.test", "samples/snippets") + + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + + # Only run system tests if found. + if os.path.exists(system_test_path) or os.path.exists(system_test_folder_path): + session.run("py.test", "tests/system") + + snippets_test_path = os.path.join("samples", "snippets") + + # Only run samples tests if found. + if os.path.exists(snippets_test_path): + session.run("py.test", "samples/snippets") diff --git a/packages/google-cloud-dlp/samples/snippets/noxfile.py b/packages/google-cloud-dlp/samples/snippets/noxfile.py index 3b3ffa5d2b0f..e9eb1cbfa5db 100644 --- a/packages/google-cloud-dlp/samples/snippets/noxfile.py +++ b/packages/google-cloud-dlp/samples/snippets/noxfile.py @@ -88,7 +88,7 @@ def get_pytest_env_vars() -> Dict[str, str]: # DO NOT EDIT - automatically generated. # All versions used to test samples. -ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9", "3.10"] +ALL_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] # Any default versions that should be ignored. IGNORED_VERSIONS = TEST_CONFIG["ignored_versions"] diff --git a/packages/google-cloud-dlp/scripts/readme-gen/templates/install_deps.tmpl.rst b/packages/google-cloud-dlp/scripts/readme-gen/templates/install_deps.tmpl.rst index 275d649890d7..6f069c6c87a5 100644 --- a/packages/google-cloud-dlp/scripts/readme-gen/templates/install_deps.tmpl.rst +++ b/packages/google-cloud-dlp/scripts/readme-gen/templates/install_deps.tmpl.rst @@ -12,7 +12,7 @@ Install Dependencies .. _Python Development Environment Setup Guide: https://cloud.google.com/python/setup -#. Create a virtualenv. Samples are compatible with Python 3.6+. +#. Create a virtualenv. Samples are compatible with Python 3.7+. .. code-block:: bash diff --git a/packages/google-cloud-dlp/setup.py b/packages/google-cloud-dlp/setup.py index ff8491d3f410..122761dec5cf 100644 --- a/packages/google-cloud-dlp/setup.py +++ b/packages/google-cloud-dlp/setup.py @@ -65,7 +65,6 @@ "License :: OSI Approved :: Apache Software License", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", @@ -78,7 +77,7 @@ namespace_packages=namespaces, install_requires=dependencies, extras_require=extras, - python_requires=">=3.6", + python_requires=">=3.7", scripts=["scripts/fixup_dlp_v2_keywords.py"], include_package_data=True, zip_safe=False,