From d327dfefdda62a91d1dd58b6a32e1b2725e7379b Mon Sep 17 00:00:00 2001 From: Christopher Wilcox Date: Wed, 28 Nov 2018 13:55:23 -0800 Subject: [PATCH] Add templates for flake8, coveragerc, noxfile, and black. (#6642) --- packages/google-cloud-dlp/.coveragerc | 11 +- packages/google-cloud-dlp/.flake8 | 1 + packages/google-cloud-dlp/noxfile.py | 152 ++++++++++++++++---------- packages/google-cloud-dlp/synth.py | 112 +++++++++---------- 4 files changed, 152 insertions(+), 124 deletions(-) diff --git a/packages/google-cloud-dlp/.coveragerc b/packages/google-cloud-dlp/.coveragerc index e98815844673..51fec440cebf 100644 --- a/packages/google-cloud-dlp/.coveragerc +++ b/packages/google-cloud-dlp/.coveragerc @@ -1,8 +1,5 @@ [run] branch = True -omit = - */gapic/* - */proto/* [report] fail_under = 100 @@ -12,6 +9,10 @@ exclude_lines = pragma: NO COVER # Ignore debug-only repr def __repr__ + # Ignore abstract methods + raise NotImplementedError omit = - */gapic/* - */proto/* + */gapic/*.py + */proto/*.py + */google-cloud-python/core/*.py + */site-packages/*.py \ No newline at end of file diff --git a/packages/google-cloud-dlp/.flake8 b/packages/google-cloud-dlp/.flake8 index 1f44a90f8195..61766fa84d02 100644 --- a/packages/google-cloud-dlp/.flake8 +++ b/packages/google-cloud-dlp/.flake8 @@ -1,4 +1,5 @@ [flake8] +ignore = E203, E266, E501, W503 exclude = # Exclude generated code. **/proto/** diff --git a/packages/google-cloud-dlp/noxfile.py b/packages/google-cloud-dlp/noxfile.py index 31bb3fcb7991..a9efc0e344ce 100644 --- a/packages/google-cloud-dlp/noxfile.py +++ b/packages/google-cloud-dlp/noxfile.py @@ -1,10 +1,12 @@ -# Copyright 2017, Google LLC All rights reserved. +# -*- coding: utf-8 -*- +# +# Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -18,89 +20,121 @@ import nox -LOCAL_DEPS = ( - os.path.join('..', 'api_core'), - os.path.join('..', 'core'), -) +LOCAL_DEPS = (os.path.join("..", "api_core"), os.path.join("..", "core")) +@nox.session(python="3.7") +def blacken(session): + """Run black. -def default(session): - """Run the unit test suite. + Format code to uniform standard. + """ + session.install("black") + session.run( + "black", + "google", + "tests", + "docs", + "--exclude", + ".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py", + ) + + +@nox.session(python="3.7") +def lint(session): + """Run linters. - This is intended to be run **without** an interpreter set, so - that the current ``python`` (on the ``PATH``) or the version of - Python corresponding to the ``nox`` binary the ``PATH`` can - run the tests. + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. """ - session.install('mock', 'pytest', 'pytest-cov', *LOCAL_DEPS) - session.install('-e', '.') + session.install("flake8", "black", *LOCAL_DEPS) + session.run( + "black", + "--check", + "google", + "tests", + "docs", + "--exclude", + ".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py", + ) + session.run("flake8", "google", "tests") + + +@nox.session(python="3.7") +def lint_setup_py(session): + """Verify that setup.py is valid (including RST check).""" + session.install("docutils", "pygments") + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + +def default(session): + # Install all test dependencies, then install this package in-place. + session.install("mock", "pytest", "pytest-cov") + for local_dep in LOCAL_DEPS: + session.install("-e", local_dep) + session.install("-e", ".") + + # Run py.test against the unit tests. session.run( - 'py.test', - '--quiet', - '--cov=google.cloud.dlp_v2', - '--cov-append', - '--cov-config=.coveragerc', - '--cov-report=', - '--cov-fail-under=97', - os.path.join('tests', 'unit'), - *session.posargs + "py.test", + "--quiet", + "--cov=google.cloud", + "--cov=tests.unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=97", + os.path.join("tests", "unit"), + *session.posargs, ) -@nox.session(python=['2.7', '3.5', '3.6', '3.7']) +@nox.session(python=["2.7", "3.5", "3.6", "3.7"]) def unit(session): """Run the unit test suite.""" - default(session) -@nox.session(python=['2.7', '3.6']) +@nox.session(python=["2.7", "3.7"]) def system(session): """Run the system test suite.""" - - if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''): - session.skip('Credentials must be set via environment variable.') + system_test_path = os.path.join("tests", "system.py") + system_test_folder_path = os.path.join("tests", "system") + # Sanity check: Only run tests if the environment variable is set. + if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""): + session.skip("Credentials must be set via environment variable") + + system_test_exists = os.path.exists(system_test_path) + system_test_folder_exists = os.path.exists(system_test_folder_path) + # Sanity check: only run tests if found. + if not system_test_exists and not system_test_folder_exists: + session.skip("System tests were not found") # Use pre-release gRPC for system tests. - session.install('--pre', 'grpcio') + session.install("--pre", "grpcio") - session.install('pytest') - session.install('-e', '.') + # Install all test dependencies, then install this package into the + # virtualenv's dist-packages. + session.install("mock", "pytest") + for local_dep in LOCAL_DEPS: + session.install("-e", local_dep) + session.install("-e", "../test_utils/") + session.install("-e", ".") - session.run('py.test', '--quiet', - os.path.join('tests', 'system'), *session.posargs) - - -@nox.session(python='3.6') -def lint(session): - """Run linters. - - Returns a failure if the linters find linting errors or sufficiently - serious code quality issues. - """ - session.install('flake8', *LOCAL_DEPS) - session.install('.') - session.run('flake8', 'google', 'tests') - - -@nox.session(python='3.6') -def lint_setup_py(session): - """Verify that setup.py is valid (including RST check).""" - session.install('docutils', 'pygments') - session.run('python', 'setup.py', 'check', '--restructuredtext', - '--strict') + # Run py.test against the system tests. + if system_test_exists: + session.run("py.test", "--quiet", system_test_path, *session.posargs) + if system_test_folder_exists: + session.run("py.test", "--quiet", system_test_folder_path, *session.posargs) -@nox.session(python='3.6') +@nox.session(python="3.7") def cover(session): """Run the final coverage report. This outputs the coverage report aggregating coverage from the unit test runs (not system test runs), and then erases coverage data. """ - session.chdir(os.path.dirname(__file__)) - session.install('coverage', 'pytest-cov') - session.run('coverage', 'report', '--show-missing', '--fail-under=100') - session.run('coverage', 'erase') + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=100") + session.run("coverage", "erase") diff --git a/packages/google-cloud-dlp/synth.py b/packages/google-cloud-dlp/synth.py index 710ccb34d7a4..97094534612d 100644 --- a/packages/google-cloud-dlp/synth.py +++ b/packages/google-cloud-dlp/synth.py @@ -23,64 +23,58 @@ gapic = gcp.GAPICGenerator() common = gcp.CommonTemplates() +# ---------------------------------------------------------------------------- +# Generate dlp GAPIC layer +# ---------------------------------------------------------------------------- library = gapic.py_library( "dlp", "v2", config_path="/google/privacy/dlp/artman_dlp_v2.yaml" ) -excludes = [ - "README.rst", - "nox.py", - "setup.py", - "docs/index.rst", -] -s.copy(library, excludes=excludes) +excludes = ["README.rst", "nox.py", "setup.py", "docs/index.rst"] +s.move(library, excludes=excludes) # Fix namespace -s.replace( - "google/**/*.py", - "google\.cloud\.privacy\.dlp_v2", - "google.cloud.dlp_v2" -) +s.replace("google/**/*.py", "google\.cloud\.privacy\.dlp_v2", "google.cloud.dlp_v2") # Add missing utf-8 marker s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "# Generated by the protocol buffer compiler. DO NOT EDIT!", - "# -*- coding: utf-8 -*-\n\g<0>" + "# -*- coding: utf-8 -*-\n\g<0>", ) # Fix unindentation of bullet list second line s.replace( "google/cloud/dlp_v2/gapic/dlp_service_client.py", "( \* .*\n )([^\s*])", - "\g<1> \g<2>" + "\g<1> \g<2>", ) s.replace( "google/cloud/dlp_v2/gapic/dlp_service_client.py", "(\s+)\*.*\n\s+::\n\n", - "\g<1> ::\n" + "\g<1> ::\n", ) # Fix raw-latex bits in storage_pb2.py -s.replace( - "google/cloud/dlp_v2/proto/storage_pb2.py", - "number regex.*\n(\s+)latex:.*\n", - "number regex \"(\\d{3}) \\d{3}-\\d{4} \"\\\n" - "\g<1>could be adjusted upwards if the area code is \\\n" -) +# s.replace( +# "google/cloud/dlp_v2/proto/storage_pb2.py", +# "number regex.*\n(\s+)latex:.*\n", +# "number regex \"(\\d\{3\}) \\d\{3\}-\\d\{4\} \"\\\n" +# "\g<1>could be adjusted upwards if the area code is \\\n" +# ) # Fix Docstrings in google/cloud/dlp_v2/proto/storage_pb2.py s.replace( "google/cloud/dlp_v2/proto/storage_pb2.py", "(hotword_regex:)\n(\s+Regular expression.*)\n", - "\g<1> \\\n\g<2> \\\n" + "\g<1> \\\n\g<2> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/storage_pb2.py", "(likelihood_adjustment:)\n", - "\g<1> \\\n" + "\g<1> \\\n", ) # Fix Docstrings in google/cloud/dlp_v2/proto/dlp_pb2.py @@ -88,86 +82,84 @@ "google/cloud/dlp_v2/proto/dlp_pb2.py", "(max_findings_per_item:)\n(\s+Max number.*)\n(\s+scanned. When.*)\n" "(\s+maximum returned is 1000.*)\n(\s+When set within.*)\n", - "\g<1> \\\n\g<2> \\\n\g<3> \\\n\g<4> \\\n\g<5> \\\n" + "\g<1> \\\n\g<2> \\\n\g<3> \\\n\g<4> \\\n\g<5> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(max_findings_per_request:)\n(\s+Max number of.*)\n(\s+When set .*)\n", - "\g<1> \\\n\g<2> \\\n\g<3> \\\n" + "\g<1> \\\n\g<2> \\\n\g<3> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(max_findings_per_info_type:)\n", - "\g<1> \\\n" + "\g<1> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(snapshot_inspect_template:)\n(\s+If run with an .*)\n", - "\g<1> \\\n\g<2> \\\n" + "\g<1> \\\n\g<2> \\\n", ) to_replace = [ - "processed_bytes:", "total_estimated_bytes:", "info_type_stats:", + "processed_bytes:", + "total_estimated_bytes:", + "info_type_stats:", "Statistics of how many instances of each info type were found", "requested_options:", ] for replace in to_replace: - s.replace( - "google/cloud/dlp_v2/proto/dlp_pb2.py", - f"({replace})\n", - "\g<1> \\\n" - ) + s.replace("google/cloud/dlp_v2/proto/dlp_pb2.py", f"({replace})\n", "\g<1> \\\n") s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(sensitive_value_frequency_lower_bound:)\n(\s+Lower bound.*)\n", - "\g<1> \\\n\g<2> \\\n" + "\g<1> \\\n\g<2> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(sensitive_value_frequency_upper_bound:)\n(\s+Upper bound.*)\n", - "\g<1> \\\n\g<2> \\\n" + "\g<1> \\\n\g<2> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(bucket_size:)\n(\s+Total number of equivalence.*)\n", - "\g<1> \\\n\g<2>\n" + "\g<1> \\\n\g<2>\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(bucket_values:)\n(\s+Sample of equivalence.*)\n", - "\g<1> \\\n\g<2> \\\n" + "\g<1> \\\n\g<2> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(offset_minutes:)\n(\s+Set only.*)\n", - "\g<1> \\\n\g<2> \\\n" + "\g<1> \\\n\g<2> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(result:)\n(\s+A summary of the outcome of this inspect job.)", - "\g<1> \\\n\g<2>" + "\g<1> \\\n\g<2>", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(storage_config:)\n(\s+The data to scan.\n)", - "\g<1> \\\n\g<2>" + "\g<1> \\\n\g<2>", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(inspect_config:)\n(\s+How and what to scan for.\n)", - "\g<1> \\\n\g<2>" + "\g<1> \\\n\g<2>", ) s.replace( @@ -175,7 +167,7 @@ "(inspect_template_name:)\n(\s+If provided, will be.*)\n" "(\s+InspectConfig.*)\n(\s+values persisted.*)\n(\s+actions:)\n" "(\s+Actions to.*)\n", - "\g<1> \\\n\g<2> \\\n\g<3> \\\n\g<4> \\\n\g<5> \\\n\g<6> \\\n" + "\g<1> \\\n\g<2> \\\n\g<3> \\\n\g<4> \\\n\g<5> \\\n\g<6> \\\n", ) s.replace( @@ -183,32 +175,32 @@ " (\s+Set of values defining the equivalence class.*)\n" " (\s+quasi-identifier.*)\n" " (\s+message. The order.*)\n", - "\g<1> \\\n\g<2> \\\n\g<3>\n" + "\g<1> \\\n\g<2> \\\n\g<3>\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", " (\s+Size of the equivalence class, for example number of rows with)\n" " (\s+the above set of values.)\n", - "\g<1> \\\n\g<2>\n" + "\g<1> \\\n\g<2>\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(equivalence_class_size_lower_bound:)\n(\s+Lower bound.*)\n", - "\g<1> \\\n\g<2> \\\n" + "\g<1> \\\n\g<2> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(equivalence_class_size_upper_bound:)\n(\s+Upper bound.*)\n", - "\g<1> \\\n\g<2> \\\n" + "\g<1> \\\n\g<2> \\\n", ) s.replace( "google/cloud/dlp_v2/proto/dlp_pb2.py", "(bucket_value_count:)\n(\s+Total number of distinct equivalence.*)\n", - "\g<1> \\\n\g<2>\n" + "\g<1> \\\n\g<2>\n", ) # Docstrings from categorical histogram bucket @@ -219,9 +211,7 @@ "(\s+bucket_size:)\n\s+(Total.*\n)" "(\s+bucket_values:)\n\s+(Sample of value.*)\n\s+(of values.*\n)" "(\s+bucket_value_count:)\n\s+(Total number.*\n)", - - "\g<1> \g<2> \g<3>\g<4> \g<5> \g<6>\g<7> \g<8>" - "\g<9> \g<10> \g<11>\g<12> \g<13>" + "\g<1> \g<2> \g<3>\g<4> \g<5> \g<6>\g<7> \g<8>" "\g<9> \g<10> \g<11>\g<12> \g<13>", ) # Fix docstrings tagged field indentation issues in dlp_pb2.py @@ -237,20 +227,20 @@ "\g<1>\g<2> \\\n\g<3>\n\g<4> \\\n\g<5> \\\n\g<6> \\\n" "\g<7> \\\n\g<8> \\\n\g<9> \\\n\g<10> \\\n\g<11> \\\n\g<12> \\\n" "\g<13> \\\n\g<14>\n\g<15> \\\n\g<16> \\\n\g<17> \\\n\g<18> \\\n" - "\g<19>\n\g<20> \\\n\g<21> \\\n" + "\g<19>\n\g<20> \\\n\g<21> \\\n", ) s.replace( - "google/cloud/dlp_v2/proto/dlp_pb2.py", - "(////////.*)\n\s+(///////////////\n)", - "\g<1> \g<2>" + "google/cloud/dlp_v2/proto/dlp_pb2.py", + "(////////.*)\n\s+(///////////////\n)", + "\g<1> \g<2>", ) # Fix Docstrings in google/cloud/dlp_v2/gapic/dlp_service_client.py s.replace( "google/cloud/dlp_v2/gapic/dlp_service_client.py", "^\s+resource was created.", - " \g<0>" + " \g<0>", ) # Fix Docstrings in google/cloud/dlp_v2/gapic/enums.py @@ -260,8 +250,10 @@ "\g<1>WHITESPACE (int): Whitespace character\n", ) -s.replace( - "google/cloud/dlp_v2/gapic/enums.py", - ".*:raw-latex:.*\n", - "", -) +s.replace("google/cloud/dlp_v2/gapic/enums.py", ".*:raw-latex:.*\n", "") + +# ---------------------------------------------------------------------------- +# Add templated files +# ---------------------------------------------------------------------------- +templated_files = common.py_library(unit_cov_level=97, cov_level=100) +s.move(templated_files)