From 1ef6136af88dc293ad67a88bd6a12420d76a3f05 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 5 Jun 2019 17:06:04 -0700 Subject: [PATCH 1/3] Expose publish retry settings --- .../cloud/pubsub_v1/gapic/publisher_client.py | 40 ++++++++++--- pubsub/noxfile.py | 46 ++++++++------- pubsub/synth.metadata | 10 ++-- pubsub/synth.py | 57 +++++++++++++++++++ 4 files changed, 117 insertions(+), 36 deletions(-) diff --git a/pubsub/google/cloud/pubsub_v1/gapic/publisher_client.py b/pubsub/google/cloud/pubsub_v1/gapic/publisher_client.py index 9bfe52ac638d..46281f3033fe 100644 --- a/pubsub/google/cloud/pubsub_v1/gapic/publisher_client.py +++ b/pubsub/google/cloud/pubsub_v1/gapic/publisher_client.py @@ -16,8 +16,11 @@ """Accesses the google.pubsub.v1 Publisher API.""" +import collections +from copy import deepcopy import functools import pkg_resources +import six import warnings from google.oauth2 import service_account @@ -44,6 +47,28 @@ _GAPIC_LIBRARY_VERSION = pkg_resources.get_distribution("google-cloud-pubsub").version +# TODO: remove conditional import after Python 2 support is dropped +if six.PY3: + from collections.abc import Mapping +else: + from collections import Mapping + + +def _merge_dict(d1, d2): + # Modifies d1 in-place to take values from d2 + # if the nested keys from d2 are present in d1. + # https://stackoverflow.com/a/10704003/4488789 + for k, v2 in d2.items(): + v1 = d1.get(k) # returns None if v1 has no such key + if v1 is None: + raise Exception("{} is not recognized by client_config".format(k)) + if isinstance(v1, Mapping) and isinstance(v2, Mapping): + _merge_dict(v1, v2) + else: + d1[k] = v2 + return d1 + + class PublisherClient(object): """ The service that an application uses to manipulate topics, and to send @@ -128,7 +153,7 @@ def __init__( This argument is mutually exclusive with providing a transport instance to ``transport``; doing so will raise an exception. - client_config (dict): DEPRECATED. A dictionary of call options for + client_config \(dict\): A dictionary of call options for each method. If not specified, the default configuration is used. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with @@ -136,15 +161,12 @@ def __init__( Generally, you only need to set this if you're developing your own client library. """ - # Raise deprecation warnings for things we want to go away. - if client_config is not None: - warnings.warn( - "The `client_config` argument is deprecated.", - PendingDeprecationWarning, - stacklevel=2, - ) + default_client_config = deepcopy(publisher_client_config.config) + + if client_config is None: + client_config = default_client_config else: - client_config = publisher_client_config.config + client_config = _merge_dict(default_client_config, client_config) if channel: warnings.warn( diff --git a/pubsub/noxfile.py b/pubsub/noxfile.py index f021e0290c80..0f528b7f3902 100644 --- a/pubsub/noxfile.py +++ b/pubsub/noxfile.py @@ -23,12 +23,6 @@ LOCAL_DEPS = (os.path.join("..", "api_core"), os.path.join("..", "core")) -BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] - -if os.path.exists("samples"): - BLACK_PATHS.append("samples") - - @nox.session(python="3.7") def lint(session): """Run linters. @@ -37,7 +31,13 @@ def lint(session): serious code quality issues. """ session.install("flake8", "black", *LOCAL_DEPS) - session.run("black", "--check", *BLACK_PATHS) + session.run( + "black", + "--check", + "google", + "tests", + "docs", + ) session.run("flake8", "google", "tests") @@ -52,7 +52,12 @@ def blacken(session): check the state of the `gcp_ubuntu_config` we use for that Kokoro run. """ session.install("black") - session.run("black", *BLACK_PATHS) + session.run( + "black", + "google", + "tests", + "docs", + ) @nox.session(python="3.7") @@ -135,24 +140,21 @@ def cover(session): session.run("coverage", "erase") - @nox.session(python="3.7") def docs(session): """Build the docs for this library.""" - session.install("-e", ".") - session.install("sphinx", "alabaster", "recommonmark") + session.install('-e', '.') + session.install('sphinx', 'alabaster', 'recommonmark') - shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + shutil.rmtree(os.path.join('docs', '_build'), ignore_errors=True) session.run( - "sphinx-build", - "-W", # warnings as errors - "-T", # show full traceback on exception - "-N", # no colors - "-b", - "html", - "-d", - os.path.join("docs", "_build", "doctrees", ""), - os.path.join("docs", ""), - os.path.join("docs", "_build", "html", ""), + 'sphinx-build', + '-W', # warnings as errors + '-T', # show full traceback on exception + '-N', # no colors + '-b', 'html', + '-d', os.path.join('docs', '_build', 'doctrees', ''), + os.path.join('docs', ''), + os.path.join('docs', '_build', 'html', ''), ) diff --git a/pubsub/synth.metadata b/pubsub/synth.metadata index 32341b4a8d79..6537b359f560 100644 --- a/pubsub/synth.metadata +++ b/pubsub/synth.metadata @@ -1,19 +1,19 @@ { - "updateTime": "2019-06-04T19:35:17.049372Z", + "updateTime": "2019-06-05T23:56:06.750497Z", "sources": [ { "generator": { "name": "artman", - "version": "0.23.0", - "dockerImage": "googleapis/artman@sha256:846102ebf7ea2239162deea69f64940443b4147f7c2e68d64b332416f74211ba" + "version": "0.23.1", + "dockerImage": "googleapis/artman@sha256:9d5cae1454da64ac3a87028f8ef486b04889e351c83bb95e83b8fab3959faed0" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "0026f4b890ed9e2388fb0573c0727defa6f5b82e", - "internalRef": "251265049" + "sha": "5487c78983f6bd5bbafa69166593826a90778a2f", + "internalRef": "251716150" } }, { diff --git a/pubsub/synth.py b/pubsub/synth.py index b568e148b060..8b052bcca4d6 100644 --- a/pubsub/synth.py +++ b/pubsub/synth.py @@ -109,6 +109,63 @@ Format is ``projects/{project}/subscriptions/{sub}``.""", ) +s.replace( + "google/cloud/pubsub_v1/gapic/publisher_client.py", + "import functools\n", + "import collections\n" + "from copy import deepcopy\n\g<0>" +) + +s.replace( + "google/cloud/pubsub_v1/gapic/publisher_client.py", + "import pkg_resources\n", + "\g<0>import six\n" +) + +s.replace( + "google/cloud/pubsub_v1/gapic/publisher_client.py", + "class PublisherClient", + """# TODO: remove conditional import after Python 2 support is dropped +if six.PY3: + from collections.abc import Mapping +else: + from collections import Mapping + + +def _merge_dict(d1, d2): + # Modifies d1 in-place to take values from d2 + # if the nested keys from d2 are present in d1. + # https://stackoverflow.com/a/10704003/4488789 + for k, v2 in d2.items(): + v1 = d1.get(k) # returns None if v1 has no such key + if v1 is None: + raise Exception("{} is not recognized by client_config".format(k)) + if isinstance(v1, Mapping) and isinstance(v2, Mapping): + _merge_dict(v1, v2) + else: + d1[k] = v2 + return d1 + \n\n\g<0>""" +) + +s.replace( + "google/cloud/pubsub_v1/gapic/publisher_client.py", + "client_config \(dict\): DEPRECATED.", + "client_config \(dict\):" +) + +s.replace( + "google/cloud/pubsub_v1/gapic/publisher_client.py", + "# Raise deprecation warnings .*\n.*\n.*\n.*\n.*\n.*\n", + """default_client_config = deepcopy(publisher_client_config.config) + + if client_config is None: + client_config = default_client_config + else: + client_config = _merge_dict(default_client_config, client_config) + """ +) + # ---------------------------------------------------------------------------- # Add templated files # ---------------------------------------------------------------------------- From 6865b9e0a7b30e856ae94eebe528abda8ec73353 Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 5 Jun 2019 17:22:59 -0700 Subject: [PATCH 2/3] remove backslashes --- pubsub/synth.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pubsub/synth.py b/pubsub/synth.py index 8b052bcca4d6..d7b8a460c9bb 100644 --- a/pubsub/synth.py +++ b/pubsub/synth.py @@ -151,14 +151,14 @@ def _merge_dict(d1, d2): s.replace( "google/cloud/pubsub_v1/gapic/publisher_client.py", "client_config \(dict\): DEPRECATED.", - "client_config \(dict\):" + "client_config (dict):" ) s.replace( "google/cloud/pubsub_v1/gapic/publisher_client.py", "# Raise deprecation warnings .*\n.*\n.*\n.*\n.*\n.*\n", """default_client_config = deepcopy(publisher_client_config.config) - + if client_config is None: client_config = default_client_config else: From 6a80be466598220e27cc69e14a4cc2bc0d74a4ae Mon Sep 17 00:00:00 2001 From: Tianzi Cai Date: Wed, 5 Jun 2019 17:33:00 -0700 Subject: [PATCH 3/3] re-ran synthtool from HEAD --- .../cloud/pubsub_v1/gapic/publisher_client.py | 2 +- pubsub/noxfile.py | 50 +++++++++---------- pubsub/synth.metadata | 2 +- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/pubsub/google/cloud/pubsub_v1/gapic/publisher_client.py b/pubsub/google/cloud/pubsub_v1/gapic/publisher_client.py index 46281f3033fe..d9698e301b07 100644 --- a/pubsub/google/cloud/pubsub_v1/gapic/publisher_client.py +++ b/pubsub/google/cloud/pubsub_v1/gapic/publisher_client.py @@ -153,7 +153,7 @@ def __init__( This argument is mutually exclusive with providing a transport instance to ``transport``; doing so will raise an exception. - client_config \(dict\): A dictionary of call options for + client_config (dict): A dictionary of call options for each method. If not specified, the default configuration is used. client_info (google.api_core.gapic_v1.client_info.ClientInfo): The client info used to send a user-agent string along with diff --git a/pubsub/noxfile.py b/pubsub/noxfile.py index 0f528b7f3902..968fb5a09bf1 100644 --- a/pubsub/noxfile.py +++ b/pubsub/noxfile.py @@ -23,6 +23,12 @@ LOCAL_DEPS = (os.path.join("..", "api_core"), os.path.join("..", "core")) +BLACK_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] + +if os.path.exists("samples"): + BLACK_PATHS.append("samples") + + @nox.session(python="3.7") def lint(session): """Run linters. @@ -31,13 +37,7 @@ def lint(session): serious code quality issues. """ session.install("flake8", "black", *LOCAL_DEPS) - session.run( - "black", - "--check", - "google", - "tests", - "docs", - ) + session.run("black", "--check", *BLACK_PATHS) session.run("flake8", "google", "tests") @@ -46,18 +46,13 @@ def blacken(session): """Run black. Format code to uniform standard. - + This currently uses Python 3.6 due to the automated Kokoro run of synthtool. That run uses an image that doesn't have 3.6 installed. Before updating this check the state of the `gcp_ubuntu_config` we use for that Kokoro run. """ session.install("black") - session.run( - "black", - "google", - "tests", - "docs", - ) + session.run("black", *BLACK_PATHS) @nox.session(python="3.7") @@ -83,7 +78,7 @@ def default(session): "--cov-append", "--cov-config=.coveragerc", "--cov-report=", - "--cov-fail-under=97", + "--cov-fail-under=0", os.path.join("tests", "unit"), *session.posargs, ) @@ -140,21 +135,24 @@ def cover(session): session.run("coverage", "erase") + @nox.session(python="3.7") def docs(session): """Build the docs for this library.""" - session.install('-e', '.') - session.install('sphinx', 'alabaster', 'recommonmark') + session.install("-e", ".") + session.install("sphinx", "alabaster", "recommonmark") - shutil.rmtree(os.path.join('docs', '_build'), ignore_errors=True) + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) session.run( - 'sphinx-build', - '-W', # warnings as errors - '-T', # show full traceback on exception - '-N', # no colors - '-b', 'html', - '-d', os.path.join('docs', '_build', 'doctrees', ''), - os.path.join('docs', ''), - os.path.join('docs', '_build', 'html', ''), + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), ) diff --git a/pubsub/synth.metadata b/pubsub/synth.metadata index 6537b359f560..26a0f82b68e7 100644 --- a/pubsub/synth.metadata +++ b/pubsub/synth.metadata @@ -1,5 +1,5 @@ { - "updateTime": "2019-06-05T23:56:06.750497Z", + "updateTime": "2019-06-06T00:31:04.007153Z", "sources": [ { "generator": {