diff --git a/.github/workflows/audit.yaml b/.github/workflows/audit.yaml index a762799b7..d552c1186 100644 --- a/.github/workflows/audit.yaml +++ b/.github/workflows/audit.yaml @@ -6,7 +6,7 @@ on: - cron: '30 2 * * *' jobs: - main: + audit: runs-on: ubuntu-20.04 name: Audit timeout-minutes: 10 @@ -19,43 +19,21 @@ jobs: - '1.13' - '1.14' - '1.15' + - master steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: ref: ${{ matrix.branch }} - - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - - run: sudo python3 -m pip install safety - - run: | - for file in $(find -name requirements.txt) - do - echo Audit ${file} - ( - cd $(dirname ${file}) && - safety check --full-report --file=requirements.txt\ - --ignore=$(cat pip-cve-ignore 2> /dev/null | sed -e 's/,/ --ignore=/g' || true) \ - - ) - done - - uses: asdf-vm/actions/install@v1 with: tool_versions: python 3.8.0 - if: always() - - run: sudo python3 -m pip install pipenv - if: always() - - run: | - asdf install python 3.5.8 - for file in $(find -name Pipfile) - do - echo Audit ${file} - ( - cd $(dirname ${file}) && - pipenv check --ignore=$(cat pipenv-cve-ignore 2> /dev/null || echo 0) - ) - done - if: always() + + - run: echo "${HOME}/.local/bin" >> ${GITHUB_PATH} + - run: python3 -m pip install --user --requirement=ci/requirements.txt + + - name: Audit + run: c2cciutils-audit --branch=${{ matrix.branch }} + env: + GITHUB_TOKEN: ${{ github.token }} diff --git a/.github/workflows/clean-dockerhub-tag.yaml b/.github/workflows/clean.yaml similarity index 53% rename from .github/workflows/clean-dockerhub-tag.yaml rename to .github/workflows/clean.yaml index c1ab23761..48c5cef15 100644 --- a/.github/workflows/clean-dockerhub-tag.yaml +++ b/.github/workflows/clean.yaml @@ -9,13 +9,17 @@ jobs: name: Clean docker hub tags timeout-minutes: 5 - env: - SUMMON_PROVIDER: /usr/local/bin/gopass steps: - - uses: actions/checkout@v1 - - uses: camptocamp/initialise-gopass-summon-action@v1 + - uses: actions/checkout@v2 + + - uses: camptocamp/initialise-gopass-summon-action@v2 with: ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}} github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}} + patterns: docker + + - run: echo "${HOME}/.local/bin" >> ${GITHUB_PATH} + - run: python3 -m pip install --user --requirement=ci/requirements.txt + - name: Clean docker hub tags - run: ci/clean-dockerhub-tag + run: c2cciutils-clean diff --git a/.github/workflows/ci.yaml b/.github/workflows/main.yaml similarity index 87% rename from .github/workflows/ci.yaml rename to .github/workflows/main.yaml index b04044ce2..97b898bde 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/main.yaml @@ -13,7 +13,6 @@ jobs: env: REDIS_URL: redis://localhost:6379 - PATH: /bin:/usr/bin:/usr/local/bin:/home/runner/.local/bin steps: - uses: actions/checkout@v2 @@ -25,6 +24,12 @@ jobs: patterns: pypi docker if: github.repository == 'camptocamp/tilecloud-chain' + - run: echo "${HOME}/.local/bin" >> ${GITHUB_PATH} + - run: python3 -m pip install --user --requirement=ci/requirements.txt + + - name: Checks + run: c2cciutils-checks + - run: touch tilecloud_chain/OpenLayers.js - name: Build @@ -54,12 +59,9 @@ jobs: - name: Check Python package run: | - sudo rm /etc/apt/sources.list.d/*.list - sudo apt update - sudo apt install --yes python3-setuptools python3-wheel python3 -m pip install --requirement=requirements-publish.txt python3 ./setup.py bdist_wheel twine check dist/* - name: Publish - run: ci/publish + run: c2cciutils-publish diff --git a/.github/workflows/rebuild-112-113.yaml b/.github/workflows/rebuild-112-113.yaml index 6e71870c1..984bdac1c 100644 --- a/.github/workflows/rebuild-112-113.yaml +++ b/.github/workflows/rebuild-112-113.yaml @@ -70,5 +70,6 @@ jobs: TOKEN: !var gs/ci/dockerhub/token ' bash -c 'envsubst < ci/docker-config.json.tmpl > ~/.docker/config.json' if: github.repository == 'camptocamp/tilecloud-chain' + - name: Publish - run: ci/publish ${{ matrix.branch }} + run: c2cciutils-publish --type=rebuild --branch=${{ matrix.branch }} diff --git a/.github/workflows/rebuild-114.yaml b/.github/workflows/rebuild-114.yaml index 4b64f07b5..b0b63d33a 100644 --- a/.github/workflows/rebuild-114.yaml +++ b/.github/workflows/rebuild-114.yaml @@ -76,4 +76,4 @@ jobs: if: github.repository == 'camptocamp/tilecloud-chain' - name: Publish - run: ci/publish ${{ matrix.branch }} + run: c2cciutils-publish --type=rebuild --branch=${{ matrix.branch }} diff --git a/.github/workflows/rebuild-115.yaml b/.github/workflows/rebuild-115.yaml index 06cb9117c..6261d6a23 100644 --- a/.github/workflows/rebuild-115.yaml +++ b/.github/workflows/rebuild-115.yaml @@ -61,11 +61,6 @@ jobs: - run: docker-compose logs if: failure() - - run: | - sudo rm /etc/apt/sources.list.d/*.list - sudo apt update - sudo apt install --yes python3-wheel python3-setuptools - - run: python3 -m pip install --requirement=requirements-publish.txt - run: | @@ -82,4 +77,4 @@ jobs: if: github.repository == 'camptocamp/tilecloud-chain' - name: Publish - run: ci/publish ${{ matrix.branch }} + run: c2cciutils-publish --type=rebuild --branch=${{ matrix.branch }} diff --git a/.github/workflows/rebuild.yaml b/.github/workflows/rebuild.yaml index 6750bb7a7..e8df4d794 100644 --- a/.github/workflows/rebuild.yaml +++ b/.github/workflows/rebuild.yaml @@ -19,10 +19,9 @@ jobs: env: REDIS_URL: redis://localhost:6379 - PATH: /bin:/usr/bin:/usr/local/bin:/home/runner/.local/bin steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: ref: ${{ matrix.branch }} @@ -32,6 +31,12 @@ jobs: github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}} patterns: pypi docker + - run: echo "${HOME}/.local/bin" >> ${GITHUB_PATH} + - run: python3 -m pip install --user --requirement=ci/requirements.txt + + - name: Checks + run: c2cciutils-checks + - run: touch tilecloud_chain/OpenLayers.js - name: Build @@ -68,4 +73,4 @@ jobs: - run: python3 -m pip install --requirement=requirements-publish.txt - name: Publish - run: ci/publish ${{ matrix.branch }} + run: c2cciutils-publish --type=rebuild --branch=${{ matrix.branch }} diff --git a/CHANGES.md b/CHANGES.md index e7fc0ff15..83bbcad8d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -38,23 +38,23 @@ Release 0.7 > - sql: AS geom FROM > ``` > - > More informations in the **Configure geom/sql** chapter. + > More information in the **Configure geom/sql** chapter. 2. Update from `optparse` to `argparse`, and some argument refactoring, use `--help` to see the new version. 3. Add support of Blackbery DB (`bsddb`). 4. The tile `server` is completely rewrite, now it support all cache, `REST` and `KVP` interface, - `GetFeatureInfo` request, and it can be used as a pyramid view or as a `WSGI` server. More informations in + `GetFeatureInfo` request, and it can be used as a pyramid view or as a `WSGI` server. More information in the **istribute the tiles** chapter. 5. Add three strategy to bypass the proxy/cache: Use the headers `Cache-Control: no-cache, no-store`, `Pragma: no-cache` (default). Use localhost in the URL and the header `Host: ` (recommended). - Add a `SALT` random argument (if the above don't work). More informations in the **Proxy/cache issue** + Add a `SALT` random argument (if the above don't work). More information in the **Proxy/cache issue** chapter. 6. Improve the dimensions usage by adding it ti the WMS requests, And add a `--dimensions` argument of `generate_tiles` to change the dimensions values. 7. Extract generate\_cost and generate\_amazon from generate\_controler. 8. Now we can creates legends, see the **Legends** chapter. 9. Now the tiles generation display generation statistics at the ends. -10. The EC2 configuration is moved in a separate structure, see README for more informations. +10. The EC2 configuration is moved in a separate structure, see README for more information. Release 0.6 ----------- diff --git a/Chaines.dia b/Chaines.dia index 4d3be0d31..70d7079bf 100644 Binary files a/Chaines.dia and b/Chaines.dia differ diff --git a/Dockerfile b/Dockerfile index 03ccec36d..b2c582258 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN \ node-carto \ optipng \ postgresql-client-12 net-tools iputils-ping \ - python3-pip python3-setuptools python3-wheel && \ + python3-pip && \ apt clean && \ rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /root/.cache/* diff --git a/README.md b/README.md index 831dbfd86..6c134fe1d 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Source: WMS, Mapnik. Optionally using an SQS queue, AWS host, SNS topic. -Destination in WMTS layout, on S3, on Berkley DB (`bsddb`), on MBTiles, or on local filesystem. +Destination in WMTS layout, on S3, on Berkeley DB (`bsddb`), on MBTiles, or on local filesystem. Features: @@ -144,10 +144,10 @@ needed to generate the capabilities. We can specify: In all case `http_url` or `http_urls` can include all attributes of this cache as `%(attribute)s`. -#### MBTiles vs Berkley DB (`bsddb`) +#### MBTiles vs Berkeley DB (`bsddb`) - Read performance: similar, eventually the MBTiles is 10% faster. -- Write performance: The Berkley DB is largely faster, about 10 times. +- Write performance: The Berkeley DB is largely faster, about 10 times. - List the tiles: the MBTiles is largely faster, but we usually don't need it. ### Configure layers @@ -622,7 +622,7 @@ There two ways to serve the tiles, with Apache configuration, or with an interna The advantage of the internal server are: -- Can distribute MBTiles or Berkley DB. +- Can distribute MBTiles or Berkeley DB. - Return `204 No Content` HTTP code in place of `404 Not Found` (or `403 Forbidden` for s3). - Can be used in KVP mode. - Can have zone per layer where are the tiles, otherwise it redirect on mapcache. @@ -790,7 +790,7 @@ The following commands can be used to know the time and cost to do generation: `--verbose` or `-v`: used to display info messages. -`--debug` or `-d`: used to display debug message, pleas use this option to report issue. With the debug mode +`--debug` or `-d`: used to display debug message, please use this option to report issue. With the debug mode we don't catch exceptions, and we don't log time messages. `--test ` or `-t `: used to generate only `` tiles, useful for test. diff --git a/ci/clean-dockerhub-tag b/ci/clean-dockerhub-tag deleted file mode 100755 index 75fd45ca0..000000000 --- a/ci/clean-dockerhub-tag +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -import json -import os -import subprocess -import sys - -import requests - - -def clean(image: str, tag: str, token: str) -> None: - print("Delete image '{image}:{tag}'.".format(image=image, tag=tag)) - - response = requests.head( - "https://hub.docker.com/v2/repositories/{image}/tags/{tag}/".format(image=image, tag=tag), - headers={"Authorization": "JWT " + token}, - ) - if response.status_code == 404: - return - if not response.ok: - print("Error checking image '{image}:{tag}' status.".format(image=image, tag=tag)) - print(response.text) - sys.exit(2) - - response = requests.delete( - "https://hub.docker.com/v2/repositories/{image}/tags/{tag}/".format(image=image, tag=tag), - headers={"Authorization": "JWT " + token}, - ) - if not response.ok: - print("Error on deleting tag: " + tag) - print(response.text) - sys.exit(2) - - -def main() -> None: - token = requests.post( - "https://hub.docker.com/v2/users/login/", - headers={"Content-Type": "application/json"}, - data=json.dumps( - { - "username": subprocess.check_output(["gopass", "gs/ci/dockerhub/username"]).decode(), - "password": subprocess.check_output(["gopass", "gs/ci/dockerhub/password"]).decode(), - } - ), - ).json()["token"] - - with open(os.environ["GITHUB_EVENT_PATH"]) as event_file: - ref = json.loads(event_file.read())["ref"] - - ref = ref.replace("/", "_") - - clean("camptocamp/tilecloud-chain", ref, token) - - -if __name__ == "__main__": - main() diff --git a/ci/publish b/ci/publish deleted file mode 100755 index 46ca91986..000000000 --- a/ci/publish +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -e - -NAME="camptocamp/tilecloud-chain" - -function publish_docker() { - local version=$1 - docker_version=$(echo "${version}" | sed -e 's/[\/:]/_/g') - - echo "Deploying image to docker hub for tag ${docker_version}" - docker tag "${NAME}" "${NAME}:${docker_version}" - docker push "${NAME}:${docker_version}" - if [ "$2" == TRUE ]; then - docker tag "${NAME}" "ghcr.io/${NAME}:${docker_version}" - docker push "ghcr.io/${NAME}:${docker_version}" - fi -} - -function publish_pypi() { - local version=$1 - - if curl --silent --fail "https://pypi.org/project/tilecloud_chain/${version}/" >/dev/null; then - echo "Already released ${version}" - exit 0 - fi - - rm -rf dist - python3 ./setup.py bdist_wheel - twine upload dist/*.whl -} - -if [ "$1" ]; then - publish_docker "$1" TRUE -elif [[ "${GITHUB_REF}" == "refs/heads/master" ]]; then - publish_docker latest TRUE -elif [[ "${GITHUB_REF}" =~ ^refs/tags/.* ]]; then - GIT_TAG=$(echo "${GITHUB_REF}" | sed 's|^refs/tags/||g') - publish_docker "${GIT_TAG}" TRUE - if [[ "${GIT_TAG}" == $(python3 setup.py --version) ]]; then - echo "Uploading version ${GIT_TAG} to pypi" - publish_pypi "${GIT_TAG}" TRUE - fi -elif [[ "${GITHUB_REF}" =~ ^refs/heads/[0-9]+\.[0-9]+$ ]]; then - GIT_BRANCH=$(echo "${GITHUB_REF}" | sed 's|^refs/heads/||g') - publish_docker "${GIT_BRANCH}" TRUE -elif [[ "${GITHUB_REF}" =~ ^refs/heads/.* ]]; then - GIT_BRANCH=$(echo "${GITHUB_REF}" | sed 's|^refs/heads/||g') - publish_docker "${GIT_BRANCH}" FALSE -else - echo "Not deploying image" -fi diff --git a/ci/requirements.txt b/ci/requirements.txt new file mode 100644 index 000000000..515d795ff --- /dev/null +++ b/ci/requirements.txt @@ -0,0 +1 @@ +c2cciutils==1.0.dev20201216094446 diff --git a/docker/mapcache-docker/mapcache.xml b/docker/mapcache-docker/mapcache.xml index 425ef3514..3c71f71de 100644 --- a/docker/mapcache-docker/mapcache.xml +++ b/docker/mapcache-docker/mapcache.xml @@ -71,4 +71,4 @@ image/jpeg report /tmp - \ No newline at end of file + diff --git a/setup.py b/setup.py index 5833f5131..550e626fd 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ version="1.16.0", description=( "Tools to generate tiles from WMS or Mapnik, to S3, " - "Berkley DB, MBTiles, or local filesystem in WMTS layout using " + "Berkeley DB, MBTiles, or local filesystem in WMTS layout using " "Amazon cloud services." ), long_description=README, diff --git a/tilecloud_chain/__init__.py b/tilecloud_chain/__init__.py index a8cc8662e..37f35ca19 100644 --- a/tilecloud_chain/__init__.py +++ b/tilecloud_chain/__init__.py @@ -30,6 +30,8 @@ from shapely.geometry import Polygon from shapely.ops import cascaded_union from shapely.wkb import loads as loads_wkb +import yaml + from tilecloud import BoundingPyramid, Tile, TileCoord, TileStore, consume from tilecloud.filter.error import LogErrors, MaximumConsecutiveErrors from tilecloud.filter.logger import Logger @@ -41,8 +43,6 @@ from tilecloud.store.redis import RedisTileStore from tilecloud.store.s3 import S3TileStore from tilecloud.store.sqs import SQSTileStore, maybe_stop -import yaml - from tilecloud_chain.multitilestore import MultiTileStore from tilecloud_chain.timedtilestore import TimedTileStoreWrapper @@ -588,13 +588,16 @@ def get_store(self, cache, layer, read_only=False): if not os.path.exists(os.path.dirname(filename)): os.makedirs(os.path.dirname(filename)) cache_tilestore = MBTilesTileStore( - sqlite3.connect(filename), content_type=layer["mime_type"], tilecoord_in_topleft=True, + sqlite3.connect(filename), + content_type=layer["mime_type"], + tilecoord_in_topleft=True, ) elif cache["type"] == "bsddb": metadata = {} for dimension in layer["dimensions"]: metadata["dimension_" + dimension["name"]] = dimension["default"] import bsddb3 as bsddb # pylint: disable=import-outside-toplevel,import-error + from tilecloud.store.bsddb import BSDDBTileStore # pylint: disable=import-outside-toplevel # on bsddb file @@ -616,10 +619,16 @@ def __call__(self): self._close_actions.append(Close(db)) - cache_tilestore = BSDDBTileStore(db, content_type=layer["mime_type"],) + cache_tilestore = BSDDBTileStore( + db, + content_type=layer["mime_type"], + ) elif cache["type"] == "filesystem": # on filesystem - cache_tilestore = FilesystemTileStore(layout, content_type=layer["mime_type"],) + cache_tilestore = FilesystemTileStore( + layout, + content_type=layer["mime_type"], + ) else: sys.exit("unknown cache type: " + cache["type"]) # pragma: no cover @@ -924,12 +933,16 @@ def init_tilecoords(self, layer): maxy += m_buffer bounding_pyramid.add( tilegrid.tilecoord( - zoom, max(minx, tilegrid.max_extent[0]), max(miny, tilegrid.max_extent[1]), + zoom, + max(minx, tilegrid.max_extent[0]), + max(miny, tilegrid.max_extent[1]), ) ) bounding_pyramid.add( tilegrid.tilecoord( - zoom, min(maxx, tilegrid.max_extent[2]), min(maxy, tilegrid.max_extent[3]), + zoom, + min(maxx, tilegrid.max_extent[2]), + min(maxy, tilegrid.max_extent[3]), ) ) diff --git a/tilecloud_chain/controller.py b/tilecloud_chain/controller.py index e3903fbbb..c81d602ef 100644 --- a/tilecloud_chain/controller.py +++ b/tilecloud_chain/controller.py @@ -261,7 +261,10 @@ def _fill_legend(gene, cache, server, base_urls): new_legend["height"] = pil_img.size[1] except Exception: # pragma: nocover logger.warning( - "Unable to read legend image '%s', with '%s'", path, repr(img), exc_info=True, + "Unable to read legend image '%s', with '%s'", + path, + repr(img), + exc_info=True, ) previous_legend = new_legend previous_resolution = resolution diff --git a/tilecloud_chain/copy_.py b/tilecloud_chain/copy_.py index 02fe85292..b96afd6a0 100644 --- a/tilecloud_chain/copy_.py +++ b/tilecloud_chain/copy_.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- +from argparse import ArgumentParser import logging import sys -from argparse import ArgumentParser from tilecloud_chain import Count, DropEmpty, HashDropper, TileGeneration, add_comon_options from tilecloud_chain.format import duration_format, size_format diff --git a/tilecloud_chain/cost.py b/tilecloud_chain/cost.py index 056e90073..82295d0ba 100644 --- a/tilecloud_chain/cost.py +++ b/tilecloud_chain/cost.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- -import logging -import sys from argparse import ArgumentParser from datetime import timedelta +import logging +import sys from tilecloud import Tile, TileStore from tilecloud_chain import Run, TileGeneration, add_comon_options diff --git a/tilecloud_chain/database_logger.py b/tilecloud_chain/database_logger.py index 0e7554036..0cbc4e4d7 100644 --- a/tilecloud_chain/database_logger.py +++ b/tilecloud_chain/database_logger.py @@ -4,9 +4,8 @@ import sys import time -import psycopg2 - from c2cwsgiutils import stats +import psycopg2 logger = logging.getLogger(__name__) diff --git a/tilecloud_chain/expiretiles.py b/tilecloud_chain/expiretiles.py index 6089f5854..fe48b8066 100644 --- a/tilecloud_chain/expiretiles.py +++ b/tilecloud_chain/expiretiles.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- -import sys from argparse import ArgumentParser +import sys import psycopg2 from shapely.geometry import MultiPolygon, Polygon @@ -16,19 +16,33 @@ def main(): description="Used to import the osm2pgsql expire-tiles file to Postgres", prog=sys.argv[0] ) parser.add_argument( - "--buffer", type=float, default=0.0, help="Extent buffer to the tiles [m], default is 0", + "--buffer", + type=float, + default=0.0, + help="Extent buffer to the tiles [m], default is 0", ) parser.add_argument( - "--simplify", type=float, default=0.0, help="Simplify the result geometry [m], default is 0", + "--simplify", + type=float, + default=0.0, + help="Simplify the result geometry [m], default is 0", ) parser.add_argument( - "--create", default=False, action="store_true", help="create the table if not exists", + "--create", + default=False, + action="store_true", + help="create the table if not exists", ) parser.add_argument( - "--delete", default=False, action="store_true", help="empty the table", + "--delete", + default=False, + action="store_true", + help="empty the table", ) parser.add_argument( - "file", metavar="FILE", help="The osm2pgsql expire-tiles file", + "file", + metavar="FILE", + help="The osm2pgsql expire-tiles file", ) parser.add_argument( "connection", @@ -39,7 +53,9 @@ def main(): ), ) parser.add_argument( - "table", metavar="TABLE", help="The PostgreSQL table to fill", + "table", + metavar="TABLE", + help="The PostgreSQL table to fill", ) parser.add_argument( "--schema", @@ -85,7 +101,9 @@ def main(): cursor.execute('DELETE FROM "{}"'.format((options.table))) geoms = [] - grid = QuadTileGrid(max_extent=(-20037508.34, -20037508.34, 20037508.34, 20037508.34),) + grid = QuadTileGrid( + max_extent=(-20037508.34, -20037508.34, 20037508.34, 20037508.34), + ) with open(options.file, "r") as f: for coord in f: extent = grid.extent(parse_tilecoord(coord), options.buffer) diff --git a/tilecloud_chain/generate.py b/tilecloud_chain/generate.py index a61f03380..f95d7b337 100644 --- a/tilecloud_chain/generate.py +++ b/tilecloud_chain/generate.py @@ -95,7 +95,8 @@ def _generate_init(self): if self._options.role in ("local", "master") and "logging" in self._gene.config: self._gene.imap( DatabaseLoggerInit( - self._gene.config["logging"], self._options is not None and self._options.daemon, + self._gene.config["logging"], + self._options is not None and self._options.daemon, ) ) diff --git a/tilecloud_chain/scaffolds/create/tilegeneration/config.yaml.mako_tmpl b/tilecloud_chain/scaffolds/create/tilegeneration/config.yaml.mako_tmpl index 737cc67d5..3e683cd1d 100644 --- a/tilecloud_chain/scaffolds/create/tilegeneration/config.yaml.mako_tmpl +++ b/tilecloud_chain/scaffolds/create/tilegeneration/config.yaml.mako_tmpl @@ -68,7 +68,7 @@ defaults: meta_size: 8 # the meta tiles buffer [default to 128] meta_buffer: 128 - # connexion an sql to get geometries (in column named geom) where we want to generate tiles + # connection an sql to get geometries (in column named geom) where we want to generate tiles # Warn: too complex result can slow down the application # connection: user=www-data password=www-data dbname= host=localhost # geoms: diff --git a/tilecloud_chain/server.py b/tilecloud_chain/server.py index 68e5077fa..643a3b3de 100644 --- a/tilecloud_chain/server.py +++ b/tilecloud_chain/server.py @@ -58,7 +58,8 @@ def init_tilegeneration(config_file): tilegeneration = TileGeneration( config_file, collections.namedtuple( - "Options", ["verbose", "debug", "quiet", "bbox", "zoom", "test", "near", "time", "geom"], + "Options", + ["verbose", "debug", "quiet", "bbox", "zoom", "test", "near", "time", "geom"], )( log_level == "verbose", log_level == "debug", @@ -157,7 +158,8 @@ def _get(self, path, **kwargs): grid=layer["grid_ref"], geoms={ layer_name: tilegeneration.get_geoms( - layer, extent=layer["bbox"] if "bbox" in layer else layer["grid_ref"]["bbox"], + layer, + extent=layer["bbox"] if "bbox" in layer else layer["grid_ref"]["bbox"], ) }, ) @@ -590,10 +592,14 @@ def main(global_config, **settings): add_mako_renderer(config, ".html") config.add_route( - "admin", "/{}/".format(tilegeneration.config["server"]["admin_path"]), request_method="GET", + "admin", + "/{}/".format(tilegeneration.config["server"]["admin_path"]), + request_method="GET", ) config.add_route( - "admin_run", "/{}/run".format(tilegeneration.config["server"]["admin_path"]), request_method="POST", + "admin_run", + "/{}/run".format(tilegeneration.config["server"]["admin_path"]), + request_method="POST", ) config.add_route("tiles", "/*path", request_method="GET") diff --git a/tilecloud_chain/tests/test_copy.py b/tilecloud_chain/tests/test_copy.py index 73ca78059..b52a0da48 100644 --- a/tilecloud_chain/tests/test_copy.py +++ b/tilecloud_chain/tests/test_copy.py @@ -62,7 +62,9 @@ def test_process(self): response.raise_for_status() with open("/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png", "wb") as out: out.write(response.content) - statinfo = os.stat("/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png",) + statinfo = os.stat( + "/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png", + ) self.assertEqual(statinfo.st_size, 755) self.assert_cmd_equals( @@ -84,5 +86,7 @@ def test_process(self): else "", empty_err=True, ) - statinfo = os.stat("/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png",) + statinfo = os.stat( + "/tmp/tiles/src/1.0.0/point_hash/default/21781/0/0/0.png", + ) self.assertEqual(statinfo.st_size, 103) diff --git a/tilecloud_chain/tests/test_expiretiles.py b/tilecloud_chain/tests/test_expiretiles.py index c03d7cb55..1bdbef2e1 100644 --- a/tilecloud_chain/tests/test_expiretiles.py +++ b/tilecloud_chain/tests/test_expiretiles.py @@ -31,7 +31,9 @@ def tearDownClass(cls): # noqa os.remove("/tmp/expired") os.remove("/tmp/expired-empty") - def test_expire_tiles(self,): + def test_expire_tiles( + self, + ): with LogCapture("tilecloud_chain", level=30) as log_capture: geom = ( "MULTIPOLYGON(((" diff --git a/tilecloud_chain/tests/test_serve.py b/tilecloud_chain/tests/test_serve.py index 1ae8c23bd..4be23b73b 100644 --- a/tilecloud_chain/tests/test_serve.py +++ b/tilecloud_chain/tests/test_serve.py @@ -772,14 +772,18 @@ def test_mbtiles_rest(self): PyramidView(request)() self.assertEqual(request.response.headers["Content-Type"], "application/xml") self.assert_result_equals( - request.response.body.decode("utf-8"), CAPABILITIES, regex=True, + request.response.body.decode("utf-8"), + CAPABILITIES, + regex=True, ) request.matchdict["path"] = ["static", "1.0.0", "WMTSCapabilities.xml"] PyramidView(request)() self.assertEqual(request.response.headers["Content-Type"], "application/xml") self.assert_result_equals( - request.response.body.decode("utf-8"), CAPABILITIES, regex=True, + request.response.body.decode("utf-8"), + CAPABILITIES, + regex=True, ) os.environ["TILE_NB_THREAD"] = tile_mbt @@ -865,14 +869,18 @@ def test_bsddb_rest(self): PyramidView(request)() self.assertEqual(request.response.headers["Content-Type"], "application/xml") self.assert_result_equals( - request.response.body.decode("utf-8"), CAPABILITIES, regex=True, + request.response.body.decode("utf-8"), + CAPABILITIES, + regex=True, ) request.matchdict["path"] = ["static", "1.0.0", "WMTSCapabilities.xml"] PyramidView(request)() self.assertEqual(request.response.headers["Content-Type"], "application/xml") self.assert_result_equals( - request.response.body.decode("utf-8"), CAPABILITIES, regex=True, + request.response.body.decode("utf-8"), + CAPABILITIES, + regex=True, ) log_capture.check() @@ -1091,7 +1099,9 @@ def start_response(p_code, p_headers): result = serve({"QUERY_STRING": "", "PATH_INFO": "/wmts/1.0.0/WMTSCapabilities.xml"}, start_response) self.assertEqual(code, "200 OK") self.assert_result_equals( - result[0].decode("utf-8"), CAPABILITIES, regex=True, + result[0].decode("utf-8"), + CAPABILITIES, + regex=True, ) os.environ["TILE_NB_THREAD"] = tile_mbt @@ -1109,6 +1119,8 @@ def test_ondemend_wmtscapabilities(self): PyramidView(request)() self.assertEqual(request.response.headers["Content-Type"], "application/xml") self.assert_result_equals( - request.response.body.decode("utf-8"), CAPABILITIES, regex=True, + request.response.body.decode("utf-8"), + CAPABILITIES, + regex=True, ) log_capture.check() diff --git a/tilecloud_chain/timedtilestore.py b/tilecloud_chain/timedtilestore.py index c827f1af0..730244284 100644 --- a/tilecloud_chain/timedtilestore.py +++ b/tilecloud_chain/timedtilestore.py @@ -1,4 +1,5 @@ from c2cwsgiutils import stats + from tilecloud import TileStore diff --git a/tilecloud_chain/views/admin.py b/tilecloud_chain/views/admin.py index 727fdd170..863990100 100644 --- a/tilecloud_chain/views/admin.py +++ b/tilecloud_chain/views/admin.py @@ -34,13 +34,13 @@ import threading from typing import List +from c2cwsgiutils.auth import auth_view, is_auth import pyramid.httpexceptions import pyramid.request from pyramid.view import view_config -import tilecloud_chain.server -from c2cwsgiutils.auth import auth_view, is_auth from tilecloud_chain.controller import get_status +import tilecloud_chain.server LOG = logging.getLogger(__name__) @@ -58,7 +58,10 @@ def run(self): env.update(os.environ) env["FRONTEND"] = "noninteractive" with subprocess.Popen( - self.command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, + self.command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, ) as process: stdout, stderr = process.communicate() if process.returncode != 0: