Use c2csiutils - publish to GitHub Container Registry

.dependabot/config.yaml

@@ -13,6 +13,12 @@ update_configs:
     ignored_updates:
       - match:
           dependency_name: none
+  - package_manager: python
+    directory: /ci
+    update_schedule: live
+    automerged_updates:
+      - match:
+          update_type: all
   - package_manager: docker
     directory: /
     update_schedule: weekly

.github/workflows/audit.yaml

@@ -20,39 +20,20 @@ jobs:
           - release_4
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
         with:
           ref: ${{ matrix.branch }}
-      - uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
 
-      - run: sudo python3 -m pip install safety
       - run: |
-          for file in $(find -name requirements.txt)
-          do
-            echo Audit ${file}
-            (
-              cd $(dirname ${file}) &&
-              safety check --full-report --file=requirements.txt\
-                --ignore=$(cat pip-cwe-ignore 2> /dev/null | sed -e 's/,/ --ignore=/g' || true) \
-
-            )
-          done
+          sudo rm /etc/apt/sources.list.d/*.list
+          sudo apt update
+          sudo apt install --yes python3-wheel
 
       - uses: asdf-vm/actions/install@v1
         with:
           tool_versions: python 3.8.0
-        if: always()
-      - run: sudo python3 -m pip install pipenv
-        if: always()
-      - run: |
-          for file in $(find -name Pipfile)
-          do
-            echo Audit ${file}
-            (
-              cd $(dirname ${file}) &&
-              pipenv check --ignore=$(cat pipenv-cwe-ignore 2> /dev/null || echo 0)
-            )
-          done
-        if: always()
+
+      - run: sudo python3 -m pip install --requirement=ci/requirements.txt
+
+      - name: Audit
+        run: c2cciutils-audit

.github/workflows/clean-dockerhub-tag.yaml → .github/workflows/clean.yaml

@@ -11,10 +11,20 @@ jobs:
     env:
       SUMMON_PROVIDER: /usr/local/bin/gopass
     steps:
-      - uses: actions/checkout@v1
-      - uses: camptocamp/initialise-gopass-summon-action@v1
+      - uses: actions/checkout@v2
+
+      - uses: camptocamp/initialise-gopass-summon-action@v2
         with:
           ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}}
           github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}}
+          patterns: docker
+
+      - run: |
+          sudo rm /etc/apt/sources.list.d/*.list
+          sudo apt update
+          sudo apt install --yes python3-wheel
+
+      - run: sudo python3 -m pip install --requirement=ci/requirements.txt
+
       - name: Clean docker hub tags
-        run: ci/clean-dockerhub-tag
+        run: c2cciutils-clean

.github/workflows/ci.yaml → .github/workflows/main.yaml

@@ -26,6 +26,16 @@ jobs:
           github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}}
           patterns: pypi docker
 
+      - run: |
+          sudo rm /etc/apt/sources.list.d/*.list
+          sudo apt update
+          sudo apt install --yes python3-wheel
+
+      - run: sudo python3 -m pip install --requirement=ci/requirements.txt
+
+      - name: Checks
+        run: c2cciutils-checks
+
       - name: Pull
         run: make pull
 
@@ -38,12 +48,5 @@ jobs:
       - name: Install GDAL
         run: docker run --rm camptocamp/c2cwsgiutils install-gdal
 
-      - name: Check Python package
-        run: |
-          sudo apt install --yes python3-setuptools python3-wheel
-          python3 -m pip install --requirement=requirements-publish.txt
-          python3 ./setup.py sdist bdist_wheel
-          twine check dist/*
-
-      - name: Release
-        run: scripts/publish ${{github.ref}}
+      - name: Publish
+        run: c2cciutils-publish

README.md

@@ -297,11 +297,11 @@ The levels are:
 The URL `{C2C_BASE_PATH}/health_check?checks=<check_name>` can be used to run the health checks on some
 checks, coma separated list.
 
-When you instanciate the `HealthCheck` class, two checks may be automatically enabled:
+When you instantiate the `HealthCheck` class, two checks may be automatically enabled:
 
 * If redis is configured, check that redis is reachable.
 * If redis is configured and the version information is available, check that the version matches
-  accross all instances.
+  across all instances.
 
 Look at the documentation of the `c2cwsgiutils.health_check.HealthCheck` class for more information.
 
@@ -322,7 +322,7 @@ command line. Usually done in the [Dockerfile](acceptance_tests/app/Dockerfile)
 ## Metrics
 
 The path `/metrics` provide some metrics for Prometheus.
-By default we have the `smap` `pss`, but we can easly add the `rss`, `size` or your custom settings:
+By default we have the `smap` `pss`, but we can easily add the `rss`, `size` or your custom settings:
 
 Example:
 ```

acceptance_tests/app/app_alembic/script.py.mako

@@ -5,8 +5,9 @@ Revises: ${down_revision | comma,n}
 Create Date: ${create_date}
 
 """
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
+
 ${imports if imports else ""}
 
 # revision identifiers, used by Alembic.

acceptance_tests/app/c2cwsgiutils_app/__init__.py

@@ -21,8 +21,7 @@ def broadcast_view():
 
 
 def main(_, **settings):
-    """ This function returns a Pyramid WSGI application.
-    """
+    """This function returns a Pyramid WSGI application."""
     config = Configurator(settings=settings, route_prefix="/api")
 
     # Initialise the broadcast view before c2cwsgiutils is initialised. This allows to test the

acceptance_tests/tests/tests/test_error.py

@@ -1,5 +1,5 @@
 def test_not_found(app_connection):
-    error = app_connection.get_json("inexistant", expected_status=404)
+    error = app_connection.get_json("inexistent", expected_status=404)
     print("error=" + repr(error))
     assert error["status"] == 404
 

c2cwsgiutils/acceptance/composition.py

@@ -12,7 +12,9 @@
 
 LOG = logging.getLogger(__name__)
 logging.basicConfig(
-    level=logging.DEBUG, format="TEST: %(asctime)-15s %(levelname)5s %(name)s %(message)s", stream=sys.stdout,
+    level=logging.DEBUG,
+    format="TEST: %(asctime)-15s %(levelname)5s %(name)s %(message)s",
+    stream=sys.stdout,
 )
 logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARN)
 
@@ -44,7 +46,9 @@ def __init__(
 
         # Setup something that redirects the docker container logs to the test output
         log_watcher = subprocess.Popen(
-            self.docker_compose + ["logs", "--follow", "--no-color"], env=env, stderr=subprocess.STDOUT,
+            self.docker_compose + ["logs", "--follow", "--no-color"],
+            env=env,
+            stderr=subprocess.STDOUT,
         )
         request.addfinalizer(log_watcher.kill)
         if os.environ.get("docker_stop", "1") == "1":
@@ -60,7 +64,8 @@ def dc(self, args: List[str], **kwargs: Any) -> str:
 
     def dc_try(self, args: List[str], **kwargs: Any) -> None:
         _try(
-            lambda: self.dc(args), **kwargs,
+            lambda: self.dc(args),
+            **kwargs,
         )
 
     def stop_all(self) -> None:
@@ -83,7 +88,8 @@ def restart(self, container: str) -> None:
 
     def run(self, container: str, *command: str, **kwargs: Dict[str, Any]) -> None:
         self.dc(
-            ["run", "--rm", container] + list(command), **kwargs,
+            ["run", "--rm", container] + list(command),
+            **kwargs,
         )
 
     def exec(self, container: str, *command: str, **kwargs: Dict[str, Any]) -> None:

c2cwsgiutils/acceptance/connection.py

@@ -1,9 +1,9 @@
-import re
 from enum import Enum
+import re
 from typing import Any, Mapping, Optional
 
-import requests
 from lxml import etree  # nosec
+import requests
 
 COLON_SPLIT_RE = re.compile(r"\s*,\s*")
 
@@ -87,7 +87,10 @@ def get_xml(
         get the given URL (relative to the root of API).
         """
         with self.session.get(
-            self.base_url + url, headers=self._merge_headers(headers, cors), stream=True, **kwargs,
+            self.base_url + url,
+            headers=self._merge_headers(headers, cors),
+            stream=True,
+            **kwargs,
         ) as r:
             check_response(r, expected_status, cache_expected=cache_expected)
             self._check_cors(cors, r)
@@ -231,7 +234,9 @@ def _merge_headers(self, headers: Optional[Mapping[str, str]], cors: bool) -> Ma
 
 
 def check_response(
-    r: requests.Response, expected_status: int = 200, cache_expected: CacheExpected = CacheExpected.DONT_CARE,
+    r: requests.Response,
+    expected_status: int = 200,
+    cache_expected: CacheExpected = CacheExpected.DONT_CARE,
 ) -> None:
     if isinstance(expected_status, tuple):
         assert r.status_code in expected_status, "status=%d\n%s" % (r.status_code, r.text)

c2cwsgiutils/db.py

@@ -137,12 +137,12 @@ def _add_tween(
 ) -> None:
     global tweens
 
-    master_paths: Iterable[Pattern[str]] = list(
-        map(RE_COMPILE, force_master)
-    ) if force_master is not None else []
-    slave_paths: Iterable[Pattern[str]] = list(
-        map(RE_COMPILE, force_slave)
-    ) if force_slave is not None else []
+    master_paths: Iterable[Pattern[str]] = (
+        list(map(RE_COMPILE, force_master)) if force_master is not None else []
+    )
+    slave_paths: Iterable[Pattern[str]] = (
+        list(map(RE_COMPILE, force_slave)) if force_slave is not None else []
+    )
 
     def db_chooser_tween_factory(
         handler: Callable[[pyramid.request.Request], Any], _registry: Any

c2cwsgiutils/debug/_listeners.py

@@ -29,7 +29,9 @@ def _dump_stacks_impl() -> Dict[str, Any]:
 
 # pylint: disable=too-many-branches
 def _dump_memory_impl(
-    limit: int, analyze_type: Optional[str], python_internals_map: bool = False,
+    limit: int,
+    analyze_type: Optional[str],
+    python_internals_map: bool = False,
 ) -> Mapping[str, Any]:
     nb_collected = [gc.collect(generation) for generation in range(3)]
     result = {

c2cwsgiutils/errors.py

@@ -6,14 +6,13 @@
 import traceback
 from typing import Any, Callable
 
-import pyramid.request
-import sqlalchemy.exc
+from c2cwsgiutils import _utils, auth
 from cornice import cors
 from pyramid.httpexceptions import HTTPError, HTTPException, HTTPRedirection, HTTPSuccessful
+import pyramid.request
+import sqlalchemy.exc
 from webob.request import DisconnectionError
 
-from c2cwsgiutils import _utils, auth
-
 DEVELOPMENT = os.environ.get("DEVELOPMENT", "0") != "0"
 DEPRECATED_CONFIG_KEY = "c2c.error_details_secret"
 DEPRECATED_ENV_KEY = "ERROR_DETAILS_SECRET"
@@ -164,7 +163,7 @@ def init(config: pyramid.config.Configurator) -> None:
         for exception in (sqlalchemy.exc.IntegrityError, sqlalchemy.exc.DataError):
             config.add_view(view=_integrity_error, context=exception, **common_options)
 
-        # We don't want to cry wolf if the user interrupted the uplad of the body
+        # We don't want to cry wolf if the user interrupted the upload of the body
         for exception in (ConnectionResetError, DisconnectionError):
             config.add_view(view=_client_interrupted_error, context=exception, **common_options)
 

c2cwsgiutils/health_check.py

@@ -4,27 +4,25 @@
 To use it, create an instance of this class in your application initialization and do a few calls to its
 methods add_db_check()
 """
+from collections import Counter
 import configparser
 import copy
+from enum import Enum
 import logging
 import os
 import re
 import subprocess
 import time
 import traceback
-from collections import Counter
 from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple, Union
 
-from enum import Enum
-
+from c2cwsgiutils import _utils, auth, broadcast, redis_utils, stats, version
 import pyramid.config
+from pyramid.httpexceptions import HTTPNotFound
 import pyramid.request
 import requests
 import sqlalchemy.engine
 import sqlalchemy.orm
-from pyramid.httpexceptions import HTTPNotFound
-
-from c2cwsgiutils import _utils, auth, broadcast, redis_utils, stats, version
 
 LOG = logging.getLogger(__name__)
 ALEMBIC_HEAD_RE = re.compile(r"^([a-f0-9]+) \(head\)\n$")
@@ -63,7 +61,7 @@ def _get_bindings(session: Any, engine_type: EngineType) -> List[sqlalchemy.engi
         return [session.c2c_ro_bind]
     if engine_type == EngineType.WRITE_ONLY:
         return [session.c2c_rw_bind]
-    raise NotImplementedError('Unhandled engine type %s' % engine_type)
+    raise NotImplementedError("Unhandled engine type %s" % engine_type)
 
 
 def _get_alembic_version(alembic_ini_path: str, name: str) -> str:
@@ -280,7 +278,7 @@ def add(name: str, func: Callable[..., Any], *args: Any) -> None:
 
     def add_version_check(self, name: str = "version", level: int = 2) -> None:
         """
-        Check that the version matches accross all instances
+        Check that the version matches across all instances
         :param name: the name of the check (defaults to "version")
         :param level: the level of the health check
         :return:

c2cwsgiutils/redis_utils.py

@@ -47,7 +47,10 @@ def _init() -> None:
     if sentinels:
         sentinels_str = [item.split(":") for item in sentinels.split(",")]
         _sentinel = redis.sentinel.Sentinel(
-            [(e[0], int(e[1])) for e in sentinels_str], decode_responses=True, db=db, **redis_options,
+            [(e[0], int(e[1])) for e in sentinels_str],
+            decode_responses=True,
+            db=db,
+            **redis_options,
         )
 
         try:

c2cwsgiutils/sentry.py

@@ -1,17 +1,16 @@
 import contextlib
 import logging
 import os
-from typing import MutableMapping, Any, Generator, Optional, Callable  # noqa  # pylint: disable=unused-import
+from typing import Any, Callable, Generator, MutableMapping, Optional  # noqa  # pylint: disable=unused-import
 
+from c2cwsgiutils import _utils
 import pyramid.config
 import sentry_sdk
 from sentry_sdk.integrations.logging import LoggingIntegration, ignore_logger
 from sentry_sdk.integrations.redis import RedisIntegration
 from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
 from sentry_sdk.integrations.wsgi import SentryWsgiMiddleware
 
-from c2cwsgiutils import _utils
-
 LOG = logging.getLogger(__name__)
 _client_setup = False
 
@@ -65,9 +64,9 @@ def init(config: Optional[pyramid.config.Configurator] = None) -> None:
 @contextlib.contextmanager
 def capture_exceptions() -> Generator[None, None, None]:
     """
-    Will send exceptions raised withing the context to Sentry.
+    Will send exceptions raised within the context to Sentry.
 
-    You don't need to use that for exception terminating the process (those not catched). Sentry does that
+    You don't need to use that for exception terminating the process (those not caught). Sentry does that
     already.
     """
     global _client_setup