diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..f7f7d14fe --- /dev/null +++ b/.editorconfig @@ -0,0 +1,25 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +indent_style = space +indent_size = 4 +trim_trailing_whitespace = true +max_line_length = 110 +quote_type = single + +[*.js] +indent_size = 2 +line_comment = // +block_comment_start = /* +block_comment = * +block_comment_end = */ + +[*.{json,yaml,yml}] +indent_size = 2 + +[Makefile] +indent_style = tab +tab_width = 2 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dacc5343b..ce880c030 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,8 @@ version: 2 updates: - package-ecosystem: pip - directory: '/' + directory: / schedule: interval: daily time: '02:00' open-pull-requests-limit: 99 - allow: - - dependency-type: direct - ignore: - - dependency-name: none - versions: - - '>= 0' diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index cc2f98f7f..7a34ecdb9 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -9,6 +9,7 @@ jobs: codeql: name: CodeQL scanning runs-on: ubuntu-20.04 + timeout-minutes: 15 steps: - name: Checkout repository diff --git a/.github/workflows/dependabot-auto-merge.yaml b/.github/workflows/dependabot-auto-merge.yaml index f41c643b6..5b5ff3f5c 100644 --- a/.github/workflows/dependabot-auto-merge.yaml +++ b/.github/workflows/dependabot-auto-merge.yaml @@ -17,6 +17,8 @@ jobs: auto-merge: name: Auto merge Dependabot updates runs-on: ubuntu-20.04 + timeout-minutes: 5 + steps: - run: echo "$ECHO" env: diff --git a/.github/workflows/ci.yaml b/.github/workflows/main.yaml similarity index 56% rename from .github/workflows/ci.yaml rename to .github/workflows/main.yaml index 3e2bba669..77af47031 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/main.yaml @@ -9,32 +9,32 @@ jobs: runs-on: ubuntu-20.04 name: Continuous integration timeout-minutes: 10 - - env: - PATH: /bin:/usr/bin:/usr/local/bin:/home/runner/.local/bin - SUMMON_PROVIDER: /usr/local/bin/gopass + if: "!startsWith(github.event.head_commit.message, '[skip ci] ')" steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 + + - uses: camptocamp/initialise-gopass-summon-action@v2 + with: + ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}} + github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}} + patterns: pypi + if: github.repository == 'camptocamp/tilecloud' - run: | sudo rm /etc/apt/sources.list.d/*.list sudo apt update - sudo apt install --yes python3-wheel python3-setuptools + sudo apt install --yes python3-wheel + + - run: sudo python3 -m pip install --requirement=ci/requirements.txt + + - name: Checks + run: c2cciutils-checks - run: python3 -m pip install --requirement=requirements-dev.txt - run: docker-compose up --exit-code-from test - - run: git --no-pager diff --check HEAD~100 - - uses: camptocamp/initialise-gopass-summon-action@v1 - with: - ci-gpg-private-key: ${{secrets.CI_GPG_PRIVATE_KEY}} - github-gopass-ci-token: ${{secrets.GOPASS_CI_GITHUB_TOKEN}} - if: github.repository == 'camptocamp/tilecloud' - - run: | - summon --yaml ' - USERNAME: !var gs/ci/pypi/username - PASSWORD: !var gs/ci/pypi/password - ' ci/publish-pypi + - name: Publish + run: c2cciutils-publish if: github.repository == 'camptocamp/tilecloud' diff --git a/bin/favicon.py b/bin/favicon.py index 42fc945ff..35a354641 100644 --- a/bin/favicon.py +++ b/bin/favicon.py @@ -26,18 +26,16 @@ def box(minx, miny, maxx, maxy): image_surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, SIZE, SIZE) context = cairo.Context(image_surface) context.scale(1, -1) -context.translate(0, - SIZE) +context.translate(0, -SIZE) for i, coord in enumerate(cloud.exterior.coords): if i == 0: - context.move_to(SCALE * coord[0] + TRANSLATE, - SCALE * coord[1] + 2 * TRANSLATE) + context.move_to(SCALE * coord[0] + TRANSLATE, SCALE * coord[1] + 2 * TRANSLATE) else: - context.line_to(SCALE * coord[0] + TRANSLATE, - SCALE * coord[1] + 2 * TRANSLATE) + context.line_to(SCALE * coord[0] + TRANSLATE, SCALE * coord[1] + 2 * TRANSLATE) context.close_path() context.set_source_rgb(1, 1, 1) context.fill_preserve() context.set_source_rgb(0, 0, 0) context.set_line_width(LINE_WIDTH) context.stroke() -image_surface.write_to_png('favicon.png') +image_surface.write_to_png("favicon.png") diff --git a/ci/config.yaml b/ci/config.yaml new file mode 100644 index 000000000..1e7728a33 --- /dev/null +++ b/ci/config.yaml @@ -0,0 +1,12 @@ +checks: + required_workflows: + clean.yaml: False + audit.yaml: False + rebuild.yaml: False + versions: + audit: False + rebuild: False + +publish: + docker: + images: [] diff --git a/ci/requirements.txt b/ci/requirements.txt new file mode 100644 index 000000000..23d53dbe7 --- /dev/null +++ b/ci/requirements.txt @@ -0,0 +1 @@ +c2cciutils==1.0.dev20201209160528 diff --git a/examples/download.py b/examples/download.py index 31858bc4d..64585d928 100755 --- a/examples/download.py +++ b/examples/download.py @@ -9,7 +9,6 @@ from tilecloud.filter.logger import Logger from tilecloud.store.boundingpyramid import BoundingPyramidTileStore - logging.basicConfig(level=logging.INFO) logger = logging.getLogger(os.path.basename(sys.argv[0])) diff --git a/examples/renderingtheworld.py b/examples/renderingtheworld.py index 02d8ecdcd..64ff0ce85 100755 --- a/examples/renderingtheworld.py +++ b/examples/renderingtheworld.py @@ -13,7 +13,6 @@ from tilecloud.store.renderingtheworld import RenderingTheWorldTileStore from tilecloud.store.wmts import WMTSTileStore - logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(os.path.basename(sys.argv[0])) diff --git a/tc-copy b/tc-copy index e2cfc5f48..4c77b76cb 100755 --- a/tc-copy +++ b/tc-copy @@ -1,6 +1,5 @@ #!/usr/bin/env python -from c2cwsgiutils import stats from itertools import imap import logging from optparse import OptionParser @@ -8,11 +7,19 @@ import os.path import random import sys +from c2cwsgiutils import stats + from tilecloud import BoundingPyramid, TileStore, consume from tilecloud.filter.benchmark import Benchmark, StatsdCountErrors, StatsdCountTiles from tilecloud.filter.consistenthash import EveryNth from tilecloud.filter.contenttype import ContentTypeAdder -from tilecloud.filter.error import DropErrors, LogErrors, MaximumConsecutiveErrors, MaximumErrorRate, MaximumErrors +from tilecloud.filter.error import ( + DropErrors, + LogErrors, + MaximumConsecutiveErrors, + MaximumErrorRate, + MaximumErrors, +) from tilecloud.filter.logger import Logger from tilecloud.filter.rate import RateLimit from tilecloud.store.boundingpyramid import BoundingPyramidTileStore @@ -21,25 +28,25 @@ from tilecloud.store.boundingpyramid import BoundingPyramidTileStore def main(argv): logger = logging.getLogger(os.path.basename(argv[0])) option_parser = OptionParser() - option_parser.add_option('--benchmark', action='store_true') - option_parser.add_option('-b', '--bounding-pyramid', metavar='BOUNDING-PYRAMID') - option_parser.add_option('--add-content-type', action='store_true') - option_parser.add_option('-i', metavar='I', type=int) - option_parser.add_option('-g', '--generate', metavar='TILE-STORE', action='append') - option_parser.add_option('--limit', metavar='N', type=int) - option_parser.add_option('-m', '--move', action='store_true') - option_parser.add_option('--maximum-consecutive-errors', metavar='N', type=int) - option_parser.add_option('--maximum-errors', metavar='N', type=int) - option_parser.add_option('--maximum-error-rate', metavar='FLOAT', type=float) - option_parser.add_option('-n', metavar='N', type=int) - option_parser.add_option('-o', '--overwrite', action='store_true') - option_parser.add_option('-r', '--rate-limit', metavar='HZ', type=float) - option_parser.add_option('--randomize', action='store_true') - option_parser.add_option('--statsd', action='store_true') - option_parser.add_option('--statsd-host', default='127.0.0.1', metavar='HOST') - option_parser.add_option('--statsd-port', default=8125, metavar='PORT', type=int) - option_parser.add_option('--statsd-prefix', default='tc-copy-', metavar='PREFIX') - option_parser.add_option('-v', '--verbose', action='store_true') + option_parser.add_option("--benchmark", action="store_true") + option_parser.add_option("-b", "--bounding-pyramid", metavar="BOUNDING-PYRAMID") + option_parser.add_option("--add-content-type", action="store_true") + option_parser.add_option("-i", metavar="I", type=int) + option_parser.add_option("-g", "--generate", metavar="TILE-STORE", action="append") + option_parser.add_option("--limit", metavar="N", type=int) + option_parser.add_option("-m", "--move", action="store_true") + option_parser.add_option("--maximum-consecutive-errors", metavar="N", type=int) + option_parser.add_option("--maximum-errors", metavar="N", type=int) + option_parser.add_option("--maximum-error-rate", metavar="FLOAT", type=float) + option_parser.add_option("-n", metavar="N", type=int) + option_parser.add_option("-o", "--overwrite", action="store_true") + option_parser.add_option("-r", "--rate-limit", metavar="HZ", type=float) + option_parser.add_option("--randomize", action="store_true") + option_parser.add_option("--statsd", action="store_true") + option_parser.add_option("--statsd-host", default="127.0.0.1", metavar="HOST") + option_parser.add_option("--statsd-port", default=8125, metavar="PORT", type=int) + option_parser.add_option("--statsd-prefix", default="tc-copy-", metavar="PREFIX") + option_parser.add_option("-v", "--verbose", action="store_true") options, args = option_parser.parse_args(argv[1:]) if options.verbose: logging.basicConfig(level=logging.INFO) @@ -54,7 +61,7 @@ def main(argv): if options.statsd: statsd_settings = { "c2c.statsd_address": options.statsd_host + ":" + options.statsd_port, - "c2c.statsd_prefix": options.statsd_prefix + "c2c.statsd_prefix": options.statsd_prefix, } stats.init_backends(statsd_settings) @@ -84,12 +91,12 @@ def main(argv): tilestream = imap(benchmark.sample(), tilestream) tilestream = input_tilestore.get(tilestream) if benchmark: - tilestream = imap(benchmark.sample('get'), tilestream) + tilestream = imap(benchmark.sample("get"), tilestream) for i, g in enumerate(generate): tilestream = g.get(tilestream) if options.benchmark: - tilestream = imap(benchmark.sample('generate-%d' % (i,)), tilestream) - tilestream = imap(LogErrors(logger, logging.ERROR, '%(tilecoord)s: %(error)s'), tilestream) + tilestream = imap(benchmark.sample("generate-%d" % (i,)), tilestream) + tilestream = imap(LogErrors(logger, logging.ERROR, "%(tilecoord)s: %(error)s"), tilestream) if options.statsd: tilestream = imap(StatsdCountErrors(), tilestream) if options.maximum_consecutive_errors: @@ -103,27 +110,34 @@ def main(argv): tilestream = imap(ContentTypeAdder(), tilestream) tilestream = output_tilestore.put(tilestream) if benchmark: - tilestream = imap(benchmark.sample('put'), tilestream) + tilestream = imap(benchmark.sample("put"), tilestream) if options.move: tilestream = input_tilestore.delete(tilestream) if benchmark: - tilestream = imap(benchmark.sample('delete'), tilestream) + tilestream = imap(benchmark.sample("delete"), tilestream) if options.verbose: - tilestream = imap(Logger(logger, logging.INFO, '%(tilecoord)s'), tilestream) + tilestream = imap(Logger(logger, logging.INFO, "%(tilecoord)s"), tilestream) if options.statsd: tilestream = imap(StatsdCountTiles(), tilestream) consume(tilestream, options.limit) finally: logging.basicConfig(level=logging.INFO) if benchmark: - keys = ['get'] - keys.extend('generate-%i' % (i,) for i in range(0, len(generate))) - keys.extend(['put', 'delete']) + keys = ["get"] + keys.extend("generate-%i" % (i,) for i in range(0, len(generate))) + keys.extend(["put", "delete"]) for key in keys: if key in benchmark.statisticss: statistics = benchmark.statisticss[key] - logger.info('%s: %s%s' % (key, statistics, ' (%.1f tiles/s)' % (1.0 / statistics.mean,) if statistics.n else '')) + logger.info( + "%s: %s%s" + % ( + key, + statistics, + " (%.1f tiles/s)" % (1.0 / statistics.mean,) if statistics.n else "", + ) + ) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/tc-delete b/tc-delete index 75efdbbbd..a729b9b1a 100755 --- a/tc-delete +++ b/tc-delete @@ -14,15 +14,13 @@ from tilecloud.store.boundingpyramid import BoundingPyramidTileStore def main(argv): logger = logging.getLogger(os.path.basename(argv[0])) option_parser = OptionParser() - option_parser.add_option('-b', '--bounding-pyramid', - metavar='BOUNDING-PYRAMID') - option_parser.add_option('-v', '--verbose', action='store_true') + option_parser.add_option("-b", "--bounding-pyramid", metavar="BOUNDING-PYRAMID") + option_parser.add_option("-v", "--verbose", action="store_true") options, args = option_parser.parse_args(argv[1:]) if options.verbose: logging.basicConfig(level=logging.INFO) if options.bounding_pyramid: - bounding_pyramid = \ - BoundingPyramid.from_string(options.bounding_pyramid) + bounding_pyramid = BoundingPyramid.from_string(options.bounding_pyramid) else: bounding_pyramid = None for arg in args: @@ -33,9 +31,9 @@ def main(argv): tilestream = tilestore.list() tilestream = tilestore.delete(tilestream) if options.verbose: - tilestream = imap(Logger(logger, logging.INFO, '%(tilecoord)s'), tilestream) + tilestream = imap(Logger(logger, logging.INFO, "%(tilecoord)s"), tilestream) consume(tilestream, None) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/tc-info b/tc-info index a9d18547f..3af555d0b 100755 --- a/tc-info +++ b/tc-info @@ -11,16 +11,15 @@ from tilecloud import BoundingPyramid, TileStore def main(argv): option_parser = OptionParser() - option_parser.add_option('-b', '--bounding-pyramid', - metavar='BOUNDING-PYRAMID') - option_parser.add_option('-r', '--relative', action='store_true') - option_parser.add_option('-t', '--tiles', - choices=('bounding-pyramid', 'completion', - 'count', 'estimate-completion', 'list')) + option_parser.add_option("-b", "--bounding-pyramid", metavar="BOUNDING-PYRAMID") + option_parser.add_option("-r", "--relative", action="store_true") + option_parser.add_option( + "-t", "--tiles", choices=("bounding-pyramid", "completion", "count", "estimate-completion", "list") + ) options, args = option_parser.parse_args(argv[1:]) for arg in args: tilestore = TileStore.load(arg) - if options.tiles == 'bounding-pyramid': + if options.tiles == "bounding-pyramid": bounding_pyramid = tilestore.bounding_pyramid if bounding_pyramid is None: bounding_pyramid = tilestore.get_cheap_bounding_pyramid() @@ -29,32 +28,33 @@ def main(argv): for z in sorted(bounding_pyramid.zs()): xbounds, ybounds = bounding_pyramid.zget(z) if options.relative: - print('%d/%d/%d:%+d/%+d' % (z, xbounds.start, - ybounds.start, - xbounds.stop - xbounds.start, - ybounds.stop - ybounds.start)) + print( + "%d/%d/%d:%+d/%+d" + % ( + z, + xbounds.start, + ybounds.start, + xbounds.stop - xbounds.start, + ybounds.stop - ybounds.start, + ) + ) else: - print('%d/%d/%d:%d/%d' % (z, xbounds.start, ybounds.start, - xbounds.stop, ybounds.stop)) - elif options.tiles == 'completion': - bounding_pyramid = \ - BoundingPyramid.from_string(options.bounding_pyramid) + print("%d/%d/%d:%d/%d" % (z, xbounds.start, ybounds.start, xbounds.stop, ybounds.stop)) + elif options.tiles == "completion": + bounding_pyramid = BoundingPyramid.from_string(options.bounding_pyramid) completed = defaultdict(set) - for tilecoord in imap(attrgetter('tilecoord'), tilestore.list()): + for tilecoord in imap(attrgetter("tilecoord"), tilestore.list()): completed[tilecoord.z].add((tilecoord.x, tilecoord.y)) for z in bounding_pyramid.zs(): numerator = len(completed[z]) dxbounds, dybounds = bounding_pyramid.zget(z) denominator = len(dxbounds) * len(dybounds) - print('%d %d/%d (%d%%)' % (z, numerator, denominator, - 100 * numerator / denominator)) - elif options.tiles == 'count': + print("%d %d/%d (%d%%)" % (z, numerator, denominator, 100 * numerator / denominator)) + elif options.tiles == "count": print(len(tilestore)) - elif options.tiles == 'estimate-completion': - bounding_pyramid = \ - BoundingPyramid.from_string(options.bounding_pyramid) - tilestore_bounding_pyramid = \ - tilestore.get_cheap_bounding_pyramid() + elif options.tiles == "estimate-completion": + bounding_pyramid = BoundingPyramid.from_string(options.bounding_pyramid) + tilestore_bounding_pyramid = tilestore.get_cheap_bounding_pyramid() for z in bounding_pyramid.zs(): try: nxbounds, nybounds = tilestore_bounding_pyramid.zget(z) @@ -63,12 +63,11 @@ def main(argv): numerator = 0 dxbounds, dybounds = bounding_pyramid.zget(z) denominator = len(dxbounds) * len(dybounds) - print('%d %d/%d (%d%%)' % (z, numerator, denominator, - 100 * numerator / denominator)) - elif options.tiles == 'list': + print("%d %d/%d (%d%%)" % (z, numerator, denominator, 100 * numerator / denominator)) + elif options.tiles == "list": for tile in tilestore.list(): print(tile.tilecoord) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/tc-mask b/tc-mask index 20d887074..f1af1281b 100755 --- a/tc-mask +++ b/tc-mask @@ -11,10 +11,10 @@ from tilecloud.store.mask import MaskTileStore def main(argv): option_parser = OptionParser() - option_parser.add_option('--bounds', metavar='BOUNDS') - option_parser.add_option('--limit', metavar='LIMIT', type=int) - option_parser.add_option('--output', metavar='FILENAME') - option_parser.add_option('-z', metavar='Z', type=int) + option_parser.add_option("--bounds", metavar="BOUNDS") + option_parser.add_option("--limit", metavar="LIMIT", type=int) + option_parser.add_option("--output", metavar="FILENAME") + option_parser.add_option("-z", metavar="Z", type=int) options, args = option_parser.parse_args(argv[1:]) assert options.output assert options.z @@ -27,13 +27,12 @@ def main(argv): tilestream = bounding_pyramid_tilestore.put(tilestream) consume(tilestream, options.limit) bounding_pyramid = bounding_pyramid_tilestore.get_bounding_pyramid() - mask_tilestore = MaskTileStore(options.z, - bounding_pyramid.zget(options.z)) + mask_tilestore = MaskTileStore(options.z, bounding_pyramid.zget(options.z)) tilestream = chain.from_iterable(ts.list() for ts in tilestores) tilestream = mask_tilestore.put(tilestream) consume(tilestream, options.limit) - mask_tilestore.save(options.output, 'PNG') + mask_tilestore.save(options.output, "PNG") -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/tc-mbtiles-create b/tc-mbtiles-create index bbca9f9ca..2a28a383b 100755 --- a/tc-mbtiles-create +++ b/tc-mbtiles-create @@ -5,8 +5,7 @@ from optparse import OptionParser import sqlite3 import sys -from tilecloud import BoundingPyramid -from tilecloud import consume +from tilecloud import BoundingPyramid, consume from tilecloud.layout.i3d import I3DTileLayout from tilecloud.layout.osm import OSMTileLayout from tilecloud.layout.wrapped import WrappedTileLayout @@ -18,35 +17,32 @@ from tilecloud.store.s3 import S3TileStore def main(argv): - tilelayouts = {'i3d': I3DTileLayout, 'osm': OSMTileLayout} - tilestores = 'filesystem log s3'.split() + tilelayouts = {"i3d": I3DTileLayout, "osm": OSMTileLayout} + tilestores = "filesystem log s3".split() option_parser = OptionParser() - option_parser.add_option('--bounds', metavar='Z1/X1/Y1:(Z2/)?X2/Y2') - option_parser.add_option('--bucket', dest='bucket_name', metavar='BUCKET') - option_parser.add_option('--layout', choices=tilelayouts.keys(), - dest='tilelayout') - option_parser.add_option('--limit', metavar='LIMIT', type=int) - option_parser.add_option('--output', default=':memory:', metavar='TILESET') - option_parser.add_option('--prefix', default='', metavar='STRING') - option_parser.add_option('--store', choices=tilestores) - option_parser.add_option('--suffix', default='', metavar='STRING') - option_parser.add_option('--name', metavar='NAME') - option_parser.add_option('--type', default='baselayer', - choices=('baselayer', 'overlay')) - option_parser.add_option('--version', metavar='VERSION') - option_parser.add_option('--description', metavar='DESCRIPTION') - option_parser.add_option('--format', metavar='FORMAT') + option_parser.add_option("--bounds", metavar="Z1/X1/Y1:(Z2/)?X2/Y2") + option_parser.add_option("--bucket", dest="bucket_name", metavar="BUCKET") + option_parser.add_option("--layout", choices=tilelayouts.keys(), dest="tilelayout") + option_parser.add_option("--limit", metavar="LIMIT", type=int) + option_parser.add_option("--output", default=":memory:", metavar="TILESET") + option_parser.add_option("--prefix", default="", metavar="STRING") + option_parser.add_option("--store", choices=tilestores) + option_parser.add_option("--suffix", default="", metavar="STRING") + option_parser.add_option("--name", metavar="NAME") + option_parser.add_option("--type", default="baselayer", choices=("baselayer", "overlay")) + option_parser.add_option("--version", metavar="VERSION") + option_parser.add_option("--description", metavar="DESCRIPTION") + option_parser.add_option("--format", metavar="FORMAT") options, args = option_parser.parse_args(argv[1:]) assert options.store tilelayout = tilelayouts[options.tilelayout]() if options.prefix or options.suffix: - tilelayout = WrappedTileLayout(tilelayout, options.prefix, - options.suffix) - if options.store == 'filesystem': + tilelayout = WrappedTileLayout(tilelayout, options.prefix, options.suffix) + if options.store == "filesystem": store = FilesystemTileStore(tilelayout) - elif options.store == 'log': + elif options.store == "log": store = LogTileStore(tilelayout, fileinput.input(args)) - elif options.store == 's3': + elif options.store == "s3": store = S3TileStore(options.bucket_name, tilelayout) else: assert False @@ -58,7 +54,7 @@ def main(argv): tilestream = store.get_all() connection = sqlite3.connect(options.output) mbtiles_tilestore = MBTilesTileStore(connection, commit=False) - for key in 'name type version description format'.split(): + for key in "name type version description format".split(): value = getattr(options, key) if value is not None: mbtiles_tilestore.metadata[key] = getattr(options, key) @@ -67,5 +63,5 @@ def main(argv): connection.commit() -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/tc-mbtiles-info b/tc-mbtiles-info index f96208ecf..07a456644 100755 --- a/tc-mbtiles-info +++ b/tc-mbtiles-info @@ -8,8 +8,8 @@ import textwrap from tilecloud.store.mbtiles import Metadata, Tiles - -BOUNDING_PYRAMID_SQL = textwrap.dedent('''\ +BOUNDING_PYRAMID_SQL = textwrap.dedent( + """\ SELECT zoom_level, MIN(tile_column), @@ -25,14 +25,14 @@ BOUNDING_PYRAMID_SQL = textwrap.dedent('''\ zoom_level ORDER BY zoom_level - ''') + """ +) def main(argv): option_parser = OptionParser() - option_parser.add_option('-m', '--metadata', action='store_true') - option_parser.add_option('-t', '--tiles', - choices=('bounding-pyramid', 'count', 'list')) + option_parser.add_option("-m", "--metadata", action="store_true") + option_parser.add_option("-t", "--tiles", choices=("bounding-pyramid", "count", "list")) options, args = option_parser.parse_args(argv[1:]) for arg in args: assert os.path.exists(arg) @@ -40,24 +40,23 @@ def main(argv): if options.metadata: metadata = Metadata(connection) for key, value in metadata.iteritems(): - print('%s: %s' % (key, value)) + print("%s: %s" % (key, value)) if options.tiles: - if options.tiles == 'count': + if options.tiles == "count": print(len(Tiles(connection))) - elif options.tiles == 'bounding-pyramid': + elif options.tiles == "bounding-pyramid": cursor = connection.cursor() cursor.execute(BOUNDING_PYRAMID_SQL) for row in cursor: if row[5] == row[6]: - extra = '' + extra = "" else: - extra = ' # %+d' % (row[5] - row[6]) - print('%d/%d/%d:%d/%d%s' % (row[0], row[1], row[2], row[3], - row[4], extra)) - elif options.tiles == 'list': + extra = " # %+d" % (row[5] - row[6]) + print("%d/%d/%d:%d/%d%s" % (row[0], row[1], row[2], row[3], row[4], extra)) + elif options.tiles == "list": for key in Tiles(connection): - print('%d/%d/%d' % (key.z, key.x, key.y)) + print("%d/%d/%d" % (key.z, key.x, key.y)) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/tc-refine-bounds b/tc-refine-bounds index df270603a..95189479d 100755 --- a/tc-refine-bounds +++ b/tc-refine-bounds @@ -29,10 +29,10 @@ def detect_edge(tilestore, z, first, last, other, horizontal, up): def main(argv): option_parser = OptionParser() - option_parser.add_option('-b', '--bounds', metavar='BOUNDING-PYRAMID') - option_parser.add_option('-r', '--relative', action='store_true') - option_parser.add_option('-s', '--seed', metavar='SEED') - option_parser.add_option('-z', '--zoom', metavar='ZOOM', type=int) + option_parser.add_option("-b", "--bounds", metavar="BOUNDING-PYRAMID") + option_parser.add_option("-r", "--relative", action="store_true") + option_parser.add_option("-s", "--seed", metavar="SEED") + option_parser.add_option("-z", "--zoom", metavar="ZOOM", type=int) options, args = option_parser.parse_args(argv[1:]) bounding_pyramid = BoundingPyramid.from_string(options.bounds) z = options.zoom @@ -50,10 +50,10 @@ def main(argv): ystart = detect_edge(tilestore, z, ybounds.start, seed.y, seed.x, False, True) ystop = detect_edge(tilestore, z, seed.y, ybounds.stop, seed.x, False, False) if options.relative: - print('%d/%d/%d:+%d/+%d' % (z, xstart, ystart, xstop - xstart, ystop - ystart)) + print("%d/%d/%d:+%d/+%d" % (z, xstart, ystart, xstop - xstart, ystop - ystart)) else: - print('%d/%d/%d:%d/%d' % (z, xstart, ystart, xstop, ystop)) + print("%d/%d/%d:%d/%d" % (z, xstart, ystart, xstop, ystop)) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(main(sys.argv)) diff --git a/tc-viewer b/tc-viewer index a0113e0d2..bf226d2bf 100755 --- a/tc-viewer +++ b/tc-viewer @@ -7,38 +7,38 @@ import sys import bottle -from tilecloud import Bounds, BoundingPyramid, Tile, TileCoord, TileStore +from tilecloud import BoundingPyramid, Bounds, Tile, TileCoord, TileStore from tilecloud.filter.contenttype import ContentTypeAdder - option_parser = OptionParser() -option_parser.add_option('--cache', action='store_true') -option_parser.add_option('--debug', action='store_true', default=False) -option_parser.add_option('--root', metavar='Z/X/Y') -option_parser.add_option('--host', default='127.0.0.1', metavar='HOST') -option_parser.add_option('--port', default=8080, metavar='PORT', type=int) -option_parser.add_option('--quiet', action='store_true', default=False) -option_parser.add_option('--max-extent', metavar='MAX-EXTENT', type=str) -option_parser.add_option('--resolutions', metavar='RESOLUTIONS', type=str) -option_parser.add_option('--server', metavar='SERVER') +option_parser.add_option("--cache", action="store_true") +option_parser.add_option("--debug", action="store_true", default=False) +option_parser.add_option("--root", metavar="Z/X/Y") +option_parser.add_option("--host", default="127.0.0.1", metavar="HOST") +option_parser.add_option("--port", default=8080, metavar="PORT", type=int) +option_parser.add_option("--quiet", action="store_true", default=False) +option_parser.add_option("--max-extent", metavar="MAX-EXTENT", type=str) +option_parser.add_option("--resolutions", metavar="RESOLUTIONS", type=str) +option_parser.add_option("--server", metavar="SERVER") options, args = option_parser.parse_args(sys.argv[1:]) if options.debug: bottle.DEBUG = True if options.root: - match = re.match(r'(\d+)/(\d+)/(\d+)\Z', options.root) + match = re.match(r"(\d+)/(\d+)/(\d+)\Z", options.root) root = TileCoord(*map(int, match.groups())) else: root = TileCoord(0, 0, 0) if options.server is None: try: - import tornado.wsgi import tornado.httpserver import tornado.ioloop - options.server = 'tornado' + import tornado.wsgi + + options.server = "tornado" id(tornado) # Suppress pyflakes warning 'tornado' imported but unused except ImportError: - options.server = 'wsgiref' + options.server = "wsgiref" cache = {} if options.cache else None max_extent = options.max_extent @@ -47,14 +47,12 @@ tilestores = [(os.path.basename(arg), TileStore.load(arg)) for arg in args] content_type_adder = ContentTypeAdder() -@bottle.route('/tiles//tiles///') +@bottle.route("/tiles//tiles///") def tile(index, z, x, y, ext): # FIXME check ext if len(tilestores) < index: bottle.abort(404) - tilecoord = TileCoord(z + root.z, - x + root.x * (1 << z), - y + root.y * (1 << z)) + tilecoord = TileCoord(z + root.z, x + root.x * (1 << z), y + root.y * (1 << z)) if cache is not None and (index, z, x, y) in cache: tile = cache[(index, z, x, y)] else: @@ -70,13 +68,13 @@ def tile(index, z, x, y, ext): if tile.content_type is not None: bottle.response.content_type = tile.content_type if tile.content_encoding is not None: - bottle.response.set_header('Content-Encoding', tile.content_encoding) - bottle.response.set_header('Access-Control-Allow-Origin', '*') + bottle.response.set_header("Content-Encoding", tile.content_encoding) + bottle.response.set_header("Access-Control-Allow-Origin", "*") bottle.response.content_length = len(tile.data) return tile.data -@bottle.route('/tiles//layersettings.json') +@bottle.route("/tiles//layersettings.json") def openwebglobe_layersettings(index): if len(tilestores) < index: bottle.abort(404) @@ -85,86 +83,88 @@ def openwebglobe_layersettings(index): if bounding_pyramid is None: bounding_pyramid = tilestore.get_cheap_bounding_pyramid() if bounding_pyramid is None: - bounding_pyramid = \ - BoundingPyramid({20: (Bounds(0, 1 << 20), Bounds(0, 1 << 20))}) + bounding_pyramid = BoundingPyramid({20: (Bounds(0, 1 << 20), Bounds(0, 1 << 20))}) maxlod = max(bounding_pyramid.zs()) xbounds, ybounds = bounding_pyramid.zget(maxlod) extent = [xbounds.start, ybounds.start, xbounds.stop, ybounds.stop] - content_type = getattr(tilestore, 'content_type', 'image/jpeg') - if content_type == 'application/json': - return dict(extent=extent, maxlod=maxlod, name=name, type='elevation') - elif content_type == 'image/jpeg' or tilestore.content_type is None: - return dict(extent=extent, format='jpg', maxlod=maxlod, name=name, - type='image') - elif content_type == 'image/png': - return dict(extent=extent, format='png', maxlod=maxlod, name=name, - type='image') + content_type = getattr(tilestore, "content_type", "image/jpeg") + if content_type == "application/json": + return dict(extent=extent, maxlod=maxlod, name=name, type="elevation") + elif content_type == "image/jpeg" or tilestore.content_type is None: + return dict(extent=extent, format="jpg", maxlod=maxlod, name=name, type="image") + elif content_type == "image/png": + return dict(extent=extent, format="png", maxlod=maxlod, name=name, type="image") else: assert False -@bottle.route('/openlayers') -@bottle.view('openlayers') +@bottle.route("/openlayers") +@bottle.view("openlayers") def openlayers(): return dict(max_extent=max_extent, resolutions=resolutions, tilestores=tilestores) -@bottle.route('/googlemaps') -@bottle.view('googlemaps') +@bottle.route("/googlemaps") +@bottle.view("googlemaps") def googlemaps(): return dict(tilestores=tilestores) -@bottle.route('/jquerygeo') -@bottle.view('jquerygeo') +@bottle.route("/jquerygeo") +@bottle.view("jquerygeo") def jquerygeo(): return dict(tilestores=tilestores) -@bottle.route('/leaflet') -@bottle.view('leaflet') +@bottle.route("/leaflet") +@bottle.view("leaflet") def leaflet(): return dict(tilestores=tilestores) -@bottle.route('/modestmaps') -@bottle.view('modestmaps') +@bottle.route("/modestmaps") +@bottle.view("modestmaps") def modestmaps(): return dict(tilestores=tilestores) -@bottle.route('/polymaps') -@bottle.view('polymaps') +@bottle.route("/polymaps") +@bottle.view("polymaps") def polymaps(): return dict(tilestores=tilestores) -@bottle.route('/openwebglobe') -@bottle.view('openwebglobe') +@bottle.route("/openwebglobe") +@bottle.view("openwebglobe") def openwebglobe(): - if 'q' in bottle.request.GET: - quality = float(bottle.request.GET.get('q')) + if "q" in bottle.request.GET: + quality = float(bottle.request.GET.get("q")) else: quality = None return dict(quality=quality, tilestores=tilestores) -@bottle.route('/favicon.ico') +@bottle.route("/favicon.ico") def favicon(): - return bottle.static_file('favicon.ico', root='static') + return bottle.static_file("favicon.ico", root="static") -@bottle.route('/static/') +@bottle.route("/static/") def static(path): - return bottle.static_file(path, root='static') + return bottle.static_file(path, root="static") -@bottle.route('/') -@bottle.view('index') +@bottle.route("/") +@bottle.view("index") def index(): - return dict(debug=bottle.request.GET.get('debug')) + return dict(debug=bottle.request.GET.get("debug")) -if __name__ == '__main__': - bottle.run(host=options.host, port=options.port, reloader=options.debug, - quiet=options.quiet, server=options.server) +if __name__ == "__main__": + bottle.run( + host=options.host, + port=options.port, + reloader=options.debug, + quiet=options.quiet, + server=options.server, + ) diff --git a/tilecloud/__init__.py b/tilecloud/__init__.py index a33062b33..6190e3b99 100644 --- a/tilecloud/__init__.py +++ b/tilecloud/__init__.py @@ -738,7 +738,9 @@ def load(name): # pragma: no cover from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.filesystem import FilesystemTileStore - return FilesystemTileStore(TemplateTileLayout(name[7:]),) + return FilesystemTileStore( + TemplateTileLayout(name[7:]), + ) if name.startswith("http://") or name.startswith("https://"): from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore @@ -746,8 +748,8 @@ def load(name): # pragma: no cover return URLTileStore((TemplateTileLayout(name),)) if name.startswith("memcached://"): from tilecloud.layout.template import TemplateTileLayout - from tilecloud.store.memcached import MemcachedTileStore from tilecloud.lib.memcached import MemcachedClient + from tilecloud.store.memcached import MemcachedTileStore server, template = name[12:].split("/", 1) host, port = server.split(":", 1) @@ -760,10 +762,11 @@ def load(name): # pragma: no cover bucket, template = name[5:].split("/", 1) return S3TileStore(bucket, TemplateTileLayout(template)) if name.startswith("sqs://"): - from tilecloud.store.sqs import SQSTileStore import boto.sqs # pylint: disable=import-error from boto.sqs.jsonmessage import JSONMessage # pylint: disable=import-error + from tilecloud.store.sqs import SQSTileStore + region_name, queue_name = name[6:].split("/", 1) connection = boto.sqs.connect_to_region(region_name) queue = connection.create_queue(queue_name) @@ -776,16 +779,19 @@ def load(name): # pragma: no cover _, ext = os.path.splitext(name) if ext == ".bsddb": import bsddb # pylint: disable=import-error + from tilecloud.store.bsddb import BSDDBTileStore return BSDDBTileStore(bsddb.hashopen(name)) if ext == ".mbtiles": import sqlite3 + from tilecloud.store.mbtiles import MBTilesTileStore return MBTilesTileStore(sqlite3.connect(name)) if ext == ".zip": import zipfile + from tilecloud.store.zip import ZipTileStore return ZipTileStore(zipfile.ZipFile(name, "a")) diff --git a/tilecloud/grid/google.py b/tilecloud/grid/google.py index 8d1542676..c40150bda 100644 --- a/tilecloud/grid/google.py +++ b/tilecloud/grid/google.py @@ -1,6 +1,5 @@ from tilecloud.grid.quad import QuadTileGrid - GoogleTileGrid = QuadTileGrid( max_extent=(-20037508.34, -20037508.34, 20037508.34, 20037508.34), tile_size=256 ) diff --git a/tilecloud/lib/wmts.py b/tilecloud/lib/wmts.py index efa368d1b..07c0f8159 100644 --- a/tilecloud/lib/wmts.py +++ b/tilecloud/lib/wmts.py @@ -51,13 +51,13 @@ def get_capabilities(layers, tile_matrix_set, wmts_gettile): dimension_default: the default dimension value dimension_values: the possible dimension value matrix_set: the matrix set name - tile_matrix_set a dict that contants the tile matrix set definition: + tile_matrix_set a dict that constants the tile matrix set definition: name: the name of the tile matrix set srs: projection like 'EPSG:21781' units: units like 'meters' resolutions: array of floats for used resolutions bbox: array of floats, the use bbox where the tiles is generated - tiles_size: the sise of the tiles (int) + tiles_size: the size of the tiles (int) yorigin: 'top' if the tiles origin is at top """ return jinja2_template( diff --git a/tilecloud/store/azure_storage_blob.py b/tilecloud/store/azure_storage_blob.py index 56578b409..8cbf8186b 100644 --- a/tilecloud/store/azure_storage_blob.py +++ b/tilecloud/store/azure_storage_blob.py @@ -2,6 +2,7 @@ import os from azure.storage.blob import BlobServiceClient + from tilecloud import Tile, TileStore LOGGER = logging.getLogger(__name__) diff --git a/tilecloud/store/log.py b/tilecloud/store/log.py index 44239f9f2..a4d2f3617 100644 --- a/tilecloud/store/log.py +++ b/tilecloud/store/log.py @@ -1,6 +1,5 @@ import re - from tilecloud import Tile, TileStore diff --git a/tilecloud/store/redis.py b/tilecloud/store/redis.py index 31a896202..5a791a381 100644 --- a/tilecloud/store/redis.py +++ b/tilecloud/store/redis.py @@ -4,9 +4,9 @@ import os import socket +from c2cwsgiutils import stats import redis.sentinel -from c2cwsgiutils import stats from tilecloud import TileStore from tilecloud.store.queue import decode_message, encode_message diff --git a/tilecloud/store/tilejson.py b/tilecloud/store/tilejson.py index 8fb934217..d0489eca8 100644 --- a/tilecloud/store/tilejson.py +++ b/tilecloud/store/tilejson.py @@ -6,9 +6,10 @@ import mimetypes import os.path import re -from urllib2 import urlopen # pylint: disable=import-error from urllib.parse import urlparse +from urllib2 import urlopen # pylint: disable=import-error + from tilecloud import BoundingPyramid from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore diff --git a/tilecloud/store/wmts.py b/tilecloud/store/wmts.py index 0aacd684d..794d0ddae 100644 --- a/tilecloud/store/wmts.py +++ b/tilecloud/store/wmts.py @@ -1,5 +1,5 @@ -from tilecloud.store.url import URLTileStore from tilecloud.layout.wmts import WMTSTileLayout +from tilecloud.store.url import URLTileStore class WMTSTileStore(URLTileStore): diff --git a/tilecloud/store/zip.py b/tilecloud/store/zip.py index 2a4d68d9e..687081164 100644 --- a/tilecloud/store/zip.py +++ b/tilecloud/store/zip.py @@ -1,8 +1,8 @@ +from collections import defaultdict +from datetime import datetime import os.path import re import zipfile -from collections import defaultdict -from datetime import datetime from tilecloud import Tile, TileStore from tilecloud.layout.osm import OSMTileLayout diff --git a/tilecloud/tests/test_metatile.py b/tilecloud/tests/test_metatile.py index 7d4f728ea..27ff5a7e9 100644 --- a/tilecloud/tests/test_metatile.py +++ b/tilecloud/tests/test_metatile.py @@ -1,5 +1,5 @@ -import unittest from io import BytesIO +import unittest from PIL import Image diff --git a/tilecloud/tests/test_redis.py b/tilecloud/tests/test_redis.py index bced0c0a7..3bb76cb3f 100644 --- a/tilecloud/tests/test_redis.py +++ b/tilecloud/tests/test_redis.py @@ -1,7 +1,8 @@ import os +import time + import pytest import redis -import time from tilecloud import Tile, TileCoord from tilecloud.store.redis import RedisTileStore diff --git a/tiles/debug/black.py b/tiles/debug/black.py index 27c14d595..ee67dbc6d 100644 --- a/tiles/debug/black.py +++ b/tiles/debug/black.py @@ -1,4 +1,3 @@ from tilecloud.store.debug import DebugTileStore - tilestore = DebugTileStore(color=(0, 0, 0)) diff --git a/tiles/debug/white.py b/tiles/debug/white.py index 812dcb694..9d6ff3079 100644 --- a/tiles/debug/white.py +++ b/tiles/debug/white.py @@ -1,4 +1,3 @@ from tilecloud.store.debug import DebugTileStore - tilestore = DebugTileStore(color=(255, 255, 255)) diff --git a/tiles/mapquest_com.py b/tiles/mapquest_com.py index 028f34396..499bf447f 100644 --- a/tiles/mapquest_com.py +++ b/tiles/mapquest_com.py @@ -1,7 +1,6 @@ from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore - tilestore = URLTileStore( ( TemplateTileLayout("http://otile{0:d}.mqcdn.com/tiles/1.0.0/osm/%(z)d/%(x)d/%(y)d.png".format(i)) diff --git a/tiles/medford_buildings.py b/tiles/medford_buildings.py index 0f170ac0e..a03ff1da8 100644 --- a/tiles/medford_buildings.py +++ b/tiles/medford_buildings.py @@ -1,6 +1,5 @@ from tilecloud.store.wmts import WMTSTileStore - tilestore = WMTSTileStore( url="http://v2.suite.opengeo.org/geoserver/gwc/service/wmts/", layer="medford:buildings", diff --git a/tiles/openstreetmap_org.py b/tiles/openstreetmap_org.py index c7f4d9cf2..ec392b09c 100644 --- a/tiles/openstreetmap_org.py +++ b/tiles/openstreetmap_org.py @@ -1,7 +1,6 @@ from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore - tilestore = URLTileStore( ( TemplateTileLayout("http://{0!s}.tile.openstreetmap.org/%(z)d/%(x)d/%(y)d.png".format(server)) diff --git a/tiles/stamen_com/terrain.py b/tiles/stamen_com/terrain.py index afa76cb8c..40a4f818c 100644 --- a/tiles/stamen_com/terrain.py +++ b/tiles/stamen_com/terrain.py @@ -1,7 +1,6 @@ from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore - tilestore = URLTileStore( ( TemplateTileLayout("http://{0!s}.tile.stamen.com/terrain/%(z)d/%(x)d/%(y)d.jpg".format(server)) diff --git a/tiles/stamen_com/toner.py b/tiles/stamen_com/toner.py index 084fd0af6..9f657fab7 100644 --- a/tiles/stamen_com/toner.py +++ b/tiles/stamen_com/toner.py @@ -1,7 +1,6 @@ from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore - tilestore = URLTileStore( ( TemplateTileLayout("http://{0!s}.tile.stamen.com/toner/%(z)d/%(x)d/%(y)d.png".format(server)) diff --git a/tiles/stamen_com/watercolor.py b/tiles/stamen_com/watercolor.py index 69904eeb4..793379d9a 100644 --- a/tiles/stamen_com/watercolor.py +++ b/tiles/stamen_com/watercolor.py @@ -1,7 +1,6 @@ from tilecloud.layout.template import TemplateTileLayout from tilecloud.store.url import URLTileStore - tilestore = URLTileStore( ( TemplateTileLayout("http://{0!s}.tile.stamen.com/watercolor/%(z)d/%(x)d/%(y)d.jpg".format(server))