Skip to content

Commit

Permalink
Add templates for flake8, coveragerc, noxfile, and black. (#6642)
Browse files Browse the repository at this point in the history
  • Loading branch information
crwilcox authored Nov 28, 2018
1 parent 97f4be0 commit 4103c45
Show file tree
Hide file tree
Showing 4 changed files with 155 additions and 57 deletions.
18 changes: 18 additions & 0 deletions packages/google-cloud-dataproc/.coveragerc
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
[run]
branch = True

[report]
fail_under = 100
show_missing = True
exclude_lines =
# Re-enable the standard pragma
pragma: NO COVER
# Ignore debug-only repr
def __repr__
# Ignore abstract methods
raise NotImplementedError
omit =
*/gapic/*.py
*/proto/*.py
*/google-cloud-python/core/*.py
*/site-packages/*.py
13 changes: 13 additions & 0 deletions packages/google-cloud-dataproc/.flake8
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
[flake8]
ignore = E203, E266, E501, W503
exclude =
# Exclude generated code.
**/proto/**
**/gapic/**
*_pb2.py

# Standard linting exemptions.
__pycache__,
.git,
*.pyc,
conf.py
135 changes: 97 additions & 38 deletions packages/google-cloud-dataproc/noxfile.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
# Copyright 2017 Google LLC
# -*- coding: utf-8 -*-
#
# Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
Expand All @@ -13,69 +15,126 @@
# limitations under the License.

from __future__ import absolute_import

import os

import nox


LOCAL_DEPS = (
os.path.join('..', 'api_core'),
)
LOCAL_DEPS = (os.path.join("..", "api_core"), os.path.join("..", "core"))

@nox.session(python="3.7")
def blacken(session):
"""Run black.
def default(session):
"""Default unit test session.
Format code to uniform standard.
"""
session.install("black")
session.run(
"black",
"google",
"tests",
"docs",
"--exclude",
".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py",
)

This is intended to be run **without** an interpreter set, so
that the current ``python`` (on the ``PATH``) or the version of
Python corresponding to the ``nox`` binary on the ``PATH`` can
run the tests.

@nox.session(python="3.7")
def lint(session):
"""Run linters.
Returns a failure if the linters find linting errors or sufficiently
serious code quality issues.
"""
session.install("flake8", "black", *LOCAL_DEPS)
session.run(
"black",
"--check",
"google",
"tests",
"docs",
"--exclude",
".*/proto/.*|.*/gapic/.*|.*/.*_pb2.py",
)
session.run("flake8", "google", "tests")


@nox.session(python="3.7")
def lint_setup_py(session):
"""Verify that setup.py is valid (including RST check)."""
session.install("docutils", "pygments")
session.run("python", "setup.py", "check", "--restructuredtext", "--strict")


def default(session):
# Install all test dependencies, then install this package in-place.
session.install('mock', 'pytest', 'pytest-cov', *LOCAL_DEPS)
session.install('-e', '.')
session.install("mock", "pytest", "pytest-cov")
for local_dep in LOCAL_DEPS:
session.install("-e", local_dep)
session.install("-e", ".")

# Run py.test against the unit tests.
session.run(
'py.test',
'--quiet',
'--cov=google.cloud.dataproc',
'--cov=google.cloud.dataproc_v1',
'--cov=tests.unit',
'--cov-append',
'--cov-config=.coveragerc',
'--cov-report=',
os.path.join('tests', 'unit'),
*session.posargs
"py.test",
"--quiet",
"--cov=google.cloud",
"--cov=tests.unit",
"--cov-append",
"--cov-config=.coveragerc",
"--cov-report=",
"--cov-fail-under=97",
os.path.join("tests", "unit"),
*session.posargs,
)


@nox.session(python=['2.7', '3.5', '3.6', '3.7'])
@nox.session(python=["2.7", "3.5", "3.6", "3.7"])
def unit(session):
"""Run the unit test suite."""
default(session)


@nox.session(python=['2.7', '3.7'])
@nox.session(python=["2.7", "3.7"])
def system(session):
"""Run the system test suite."""
system_test_path = os.path.join("tests", "system.py")
system_test_folder_path = os.path.join("tests", "system")
# Sanity check: Only run tests if the environment variable is set.
if not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", ""):
session.skip("Credentials must be set via environment variable")

system_test_exists = os.path.exists(system_test_path)
system_test_folder_exists = os.path.exists(system_test_folder_path)
# Sanity check: only run tests if found.
if not system_test_exists and not system_test_folder_exists:
session.skip("System tests were not found")

if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''):
session.skip('Credentials must be set via environment variable.')
# Use pre-release gRPC for system tests.
session.install('--pre', 'grpcio')
session.install("--pre", "grpcio")

session.install('pytest')
session.install('-e', '.')
# Install all test dependencies, then install this package into the
# virtualenv's dist-packages.
session.install("mock", "pytest")
for local_dep in LOCAL_DEPS:
session.install("-e", local_dep)
session.install("-e", "../test_utils/")
session.install("-e", ".")

session.run('py.test', '--quiet', os.path.join('tests', 'system'),
*session.posargs)
# Run py.test against the system tests.
if system_test_exists:
session.run("py.test", "--quiet", system_test_path, *session.posargs)
if system_test_folder_exists:
session.run("py.test", "--quiet", system_test_folder_path, *session.posargs)


@nox.session(python='3.6')
def lint_setup_py(session):
"""Verify that setup.py is valid (including RST check)."""
session.install('docutils', 'pygments')
session.run('python', 'setup.py', 'check', '--restructuredtext',
'--strict')
@nox.session(python="3.7")
def cover(session):
"""Run the final coverage report.
This outputs the coverage report aggregating coverage from the unit
test runs (not system test runs), and then erases coverage data.
"""
session.install("coverage", "pytest-cov")
session.run("coverage", "report", "--show-missing", "--fail-under=100")

session.run("coverage", "erase")
46 changes: 27 additions & 19 deletions packages/google-cloud-dataproc/synth.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,26 +20,27 @@
from synthtool import gcp

gapic = gcp.GAPICGenerator()
common = gcp.CommonTemplates()
versions = ["v1beta2", "v1"]

versions = ['v1beta2', 'v1']


# ----------------------------------------------------------------------------
# Generate dataproc GAPIC layer
# ----------------------------------------------------------------------------
for version in versions:
library = gapic.py_library('dataproc', version)
s.move(
library,
excludes=[
'docs/index.rst', 'nox.py', 'README.rst', 'setup.py'])
library = gapic.py_library("dataproc", version)
s.move(library, excludes=["docs/index.rst", "nox.py", "README.rst", "setup.py"])

s.replace(
f'google/cloud/dataproc_{version}/gapic/cluster_controller_client.py',
'metadata_type=operations_pb2.ClusterOperationMetadata,',
'metadata_type=proto_operations_pb2.ClusterOperationMetadata,')
f"google/cloud/dataproc_{version}/gapic/cluster_controller_client.py",
"metadata_type=operations_pb2.ClusterOperationMetadata,",
"metadata_type=proto_operations_pb2.ClusterOperationMetadata,",
)

s.replace(
f'google/cloud/dataproc_{version}/gapic/cluster_controller_client.py',
'\s+Note:.*\n(.*\n)+?.*types.FieldMask.',
f'''
f"google/cloud/dataproc_{version}/gapic/cluster_controller_client.py",
"\s+<strong>Note:</strong>.*\n(.*\n)+?.*types.FieldMask.",
f"""
.. note::
Expand All @@ -52,7 +53,7 @@
secondary worker group
If a dict is provided, it must be of the same form as the protobuf
message :class:`~google.cloud.dataproc_{version}.types.FieldMask`'''
message :class:`~google.cloud.dataproc_{version}.types.FieldMask`""",
)

s.replace(
Expand All @@ -71,7 +72,14 @@
)

s.replace(
'google/cloud/dataproc_v1beta2/proto/clusters_pb2.py',
'# Generated by the protocol buffer compiler. DO NOT EDIT!',
'# -*- coding: utf-8 -*-\n'
'# Generated by the protocol buffer compiler. DO NOT EDIT!')
"google/cloud/dataproc_v1beta2/proto/clusters_pb2.py",
"# Generated by the protocol buffer compiler. DO NOT EDIT!",
"# -*- coding: utf-8 -*-\n"
"# Generated by the protocol buffer compiler. DO NOT EDIT!",
)

# ----------------------------------------------------------------------------
# Add templated files
# ----------------------------------------------------------------------------
templated_files = common.py_library(unit_cov_level=97, cov_level=100)
s.move(templated_files)

0 comments on commit 4103c45

Please sign in to comment.