Parse the WMS capabilities less times.

One time per process and OGC server, Not one time per layers. Fix the CI
camptocamp · Jun 15, 2018 · d15b57a · d15b57a
1 parent 65cbad7
commit d15b57a
Show file tree

Hide file tree

Showing 9 changed files with 40 additions and 15 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -96,8 +96,6 @@ dockerBuild {
                     sh '(cd ${HOME}/workspace/testgeomapfish/; git commit -m "Upgrade the po files")'
                     sh '(cd ${HOME}/workspace/testgeomapfish/; ./docker-run travis/empty-make --makefile=travis.mk help)'
                     sh '(cd ${HOME}/workspace/testgeomapfish/; ./docker-run make --makefile=travis.mk build)'
-                    sh 'cat ${HOME}/workspace/testgeomapfish/testdb/*.sql'
-                    sh 'cat ${HOME}/workspace/testgeomapfish/geoportal/config.yaml'
                     withCredentials([[
                         $class: 'UsernamePasswordMultiBinding',
                         credentialsId: 'dockerhub',
@@ -106,9 +104,10 @@ dockerBuild {
                     ]]) {
                         try {
                             sh 'docker login -u "$USERNAME" -p "$PASSWORD"'
-                            sh '(cd ${HOME}/workspace/testgeomapfish/; docker-compose up --force-recreate -d)'
+                            sh '(cd ${HOME}/workspace/testgeomapfish/; docker-compose up -d)'
                             timeout(time: 2, unit: 'MINUTES') {
                                 sh '(cd ${HOME}/workspace/testgeomapfish/; docker-compose exec -T geoportal wait-for-db)'
+                                sh '(cd ${HOME}/workspace/testgeomapfish/; docker-compose exec -T geoportal create-demo-theme)'
                                 sh './docker-run travis/waitwsgi http://`netstat --route --numeric|grep ^0.0.0.0|awk \'{print($2)}\'`:8080/'
                             }
                             for (path in [

diff --git a/geoportal/c2cgeoportal_geoportal/scaffolds/create/docker-compose-build.yaml.mako b/geoportal/c2cgeoportal_geoportal/scaffolds/create/docker-compose-build.yaml.mako
@@ -32,7 +32,7 @@ ${service_defaults('mapserver', 80)}\
       - run
     links:
       - mapserver
-${service_defaults('geoportal', 80)}\
+${service_defaults('geoportal-build', 80)}\
       - HOME_DIR
       - USER_NAME
       - USER_ID

diff --git a/geoportal/c2cgeoportal_geoportal/scaffolds/update/CONST.mako_inc b/geoportal/c2cgeoportal_geoportal/scaffolds/update/CONST.mako_inc
@@ -1,5 +1,10 @@
 <%def name="service_defaults(service, inner_port=None, port_required=False)">\
 % if port_required or 'port' in docker_services.get(service, {}):
+<%
+if 'port' not in docker_services[service]:
+    raise Exception("Required port is not configured for service {}.".format(service))
+endif
+%>
     ports:
       - ${docker_services[service]['port']}:${inner_port}
 % endif

diff --git a/geoportal/c2cgeoportal_geoportal/scaffolds/update/CONST_vars.yaml_tmpl b/geoportal/c2cgeoportal_geoportal/scaffolds/update/CONST_vars.yaml_tmpl
@@ -46,10 +46,8 @@ vars:
         POSTGRES_DB: geomapfish
         POSTGRES_USER: www-data
         POSTGRES_PASSWORD: www-data
-    geoportal:
-      environment:
-        VISIBLE_WEB_HOST: '{docker_host}'
-        VISIBLE_WEB_PROTOCOL: '{docker_web_protocol}'
+    geoportal-build:
+      environment: &geo-run-env
         VISIBLE_ENTRY_POINT: '{docker_entry_point}'
         PGHOST: 172.17.0.1
         PGHOST_SLAVE: 172.17.0.1
@@ -59,11 +57,18 @@ vars:
         PGDATABASE: geomapfish
         PGSCHEMA: main
         PGSCHEMA_STATIC: main_static
+    geoportal:
+      environment:
+        <<: *geo-run-env
+        VISIBLE_WEB_HOST: '{docker_host}'
+        VISIBLE_WEB_PROTOCOL: '{docker_web_protocol}'
         TINYOWS_URL: http://tinyows/
         MAPSERVER_URL: http://mapserver/
         PRINT_URL: http://print:8080/print/
         REDIS_HOST: redis
         REDIS_PORT: 6379
+        C2C_REDIS_URL: redis://redis:6379
+        C2C_BROADCAST_PREFIX: c2cgeoportal_
     tilecloudchain:
       environment:
         VISIBLE_ENTRY_POINT: '{docker_entry_point}'

diff --git a/geoportal/c2cgeoportal_geoportal/scripts/create_demo_theme.py b/geoportal/c2cgeoportal_geoportal/scripts/create_demo_theme.py
@@ -45,7 +45,7 @@ def main():
     )
     parser.add_argument(
         '-i', '--iniconfig',
-        default='geoportal/production.ini',
+        default='production.ini',
         help='project .ini config file'
     )
     parser.add_argument(
@@ -91,6 +91,8 @@ def main():
 
     transaction.commit()
 
+    print("Successfully added the demo theme")
+
 
 if __name__ == "__main__":
     main()
diff --git a/geoportal/c2cgeoportal_geoportal/views/entry.py b/geoportal/c2cgeoportal_geoportal/views/entry.py
@@ -49,13 +49,15 @@
 from sqlalchemy.orm import subqueryload
 from sqlalchemy.orm.exc import NoResultFound
 from owslib.wms import WebMapService
+from typing import Dict, Tuple, Set  # noqa # pylint: disable=unused-import
+import zope.event.classhandler
 
 from c2cgeoportal_commons import models
 from c2cgeoportal_commons.models import main, static
 from c2cgeoportal_geoportal.lib import get_setting, get_protected_layers_query, \
     get_url2, get_url, get_typed, get_types_map, add_url_params
 from c2cgeoportal_geoportal.lib.cacheversion import get_cache_version
-from c2cgeoportal_geoportal.lib.caching import get_region, invalidate_region,  \
+from c2cgeoportal_geoportal.lib.caching import get_region, \
     set_common_headers, NO_CACHE, PUBLIC_CACHE, PRIVATE_CACHE
 from c2cgeoportal_geoportal.lib.functionality import get_functionality, get_mapserver_substitution_params
 from c2cgeoportal_geoportal.lib.wmstparsing import parse_extent, TimeInformation
@@ -115,6 +117,7 @@ class Entry:
     WFS_NS = "http://www.opengis.net/wfs"
     default_ogc_server = None
     external_ogc_server = None
+    server_wms_capabilities = {}  # type: Dict[int, Tuple[WebMapService, Set[str]]]
 
     def __init__(self, request):
         self.request = request
@@ -160,6 +163,13 @@ def __init__(self, request):
                     ", ".join([i[0] for i in models.DBSession.query(main.OGCServer.name).all()])
                 )
 
+        from c2cgeoportal_commons.models.main import InvalidateCacheEvent
+
+        @zope.event.classhandler.handler(InvalidateCacheEvent)
+        def handle(event: InvalidateCacheEvent):
+            del event
+            Entry.server_wms_capabilities = {}
+
     @view_config(route_name="testi18n", renderer="testi18n.html")
     def testi18n(self):  # pragma: no cover
         _ = self.request.translate
@@ -200,6 +210,9 @@ def _get_metadatas(self, item, errors):
     def _wms_getcap(self, ogc_server=None):
         ogc_server = (ogc_server or self.default_ogc_server)
 
+        if ogc_server.id in self.server_wms_capabilities:
+            return self.server_wms_capabilities[ogc_server.id]
+
         url, content, errors = self._wms_getcap_cached(
             ogc_server, self._get_capabilities_cache_role_key(ogc_server)
         )
@@ -215,6 +228,9 @@ def _wms_getcap(self, ogc_server=None):
                 "\nURL: {}\n{}".format(url, content)
             errors.add(error)
             log.error(error, exc_info=True)
+
+        self.server_wms_capabilities[ogc_server.id] = (wms, errors)
+
         return wms, errors
 
     @cache_region.cache_on_arguments()
@@ -1008,7 +1024,7 @@ def _get_functionalities(theme):
     @staticmethod
     @view_config(route_name="invalidate", renderer="json")
     def invalidate_cache():  # pragma: no cover
-        invalidate_region()
+        main.cache_invalidate_cb()
         return {
             "success": True
         }

diff --git a/travis/create-new-nondocker-project.sh b/travis/create-new-nondocker-project.sh
@@ -53,7 +53,7 @@ FINALISE=TRUE make --makefile=travis.mk build
 ./docker-run alembic --config=geoportal/alembic.ini --name=main upgrade head
 ./docker-run alembic --config=geoportal/alembic.ini --name=static upgrade head
 # Create default theme
-./docker-run /build/venv/bin/python /usr/local/bin/create-demo-theme
+./docker-run /build/venv/bin/python /usr/local/bin/create-demo-theme --iniconfig geoportal/production.ini
 ./docker-run make --makefile=travis.mk update-po
 git add geoportal/testgeomapfish_geoportal/locale/*/LC_MESSAGES/*.po
 git commit -m "Add initial localisation"

diff --git a/travis/create-new-project.sh b/travis/create-new-project.sh
@@ -29,7 +29,5 @@ git clean -fX
 
 # Build
 ./docker-run make --makefile=travis.mk build docker-build-testdb
-# Create default theme
-./docker-compose-run /build/venv/bin/python /usr/local/bin/create-demo-theme
 ./docker-compose-run make --makefile=travis.mk update-po
 ./docker-compose-run make --makefile=travis.mk theme2fts
diff --git a/travis/docker-compose-build.yaml.mako b/travis/docker-compose-build.yaml.mako
@@ -44,7 +44,7 @@ ${service_defaults('mapserver', 80)}\
       - db
       - external-db
       - mapserver
-${service_defaults('geoportal', 80, True)}\
+${service_defaults('geoportal-build', 80)}\
       - HOME_DIR
       - USER_NAME
       - USER_ID