Merge pull request #3868 from camptocamp/inegrate-qgis-plugin

Integrate qgis plugin

.gitignore

@@ -26,6 +26,7 @@
 /admin/npm-packages
 /admin/c2cgeoportal_admin/locale/
 /admin/c2cgeoportal_admin.egg-info/
+/docker/qgisserver/commons/
 /docker/admin-build/Dockerfile
 /docker/admin-build/npm-packages
 /docker/test-db/12-alembic.sql

Jenkinsfile

@@ -38,14 +38,14 @@ dockerBuild {
                 sh 'make docker-build'
                 sh 'docker run --name geomapfish-db --env=POSTGRES_USER=www-data --env=POSTGRES_PASSWORD=www-data --env=POSTGRES_DB=geomapfish --publish=5432:5432 --detach camptocamp/geomapfish-test-db'
                 sh 'travis/test-upgrade-convert.sh init ${HOME}/workspace'
+                sh './docker-run travis/status.sh'
+                sh './docker-run travis/short-make build'
             }
             stage('Tests') {
                 checkout scm
                 parallel 'Lint and test c2cgeoportal': {
-                    sh './docker-run travis/status.sh'
                     sh './docker-run travis/empty-make help'
                     sh 'bash -c "test \\"`./docker-run id`\\" == \\"uid=0(root) gid=0(root) groups=0(root)\\""'
-                    sh './docker-run travis/short-make build'
                     sh './docker-run make doc'
                     // lint
                     sh './docker-run make checks'

Makefile

@@ -107,6 +107,7 @@ docker-build:
 .PHONY: build
 build: \
 	docker-build-build \
+	docker-build-qgisserver \
 	docker-build-testdb \
 	prepare-tests
 
@@ -185,6 +186,18 @@ docker-build-build: $(shell docker-required --path . --replace-pattern='^test(.*
 		$(APPS_FILES_ALT)
 	docker build --build-arg=VERSION=$(VERSION) --tag=$(DOCKER_BASE)-build:$(MAJOR_VERSION) .
 
+docker/qgisserver/commons: commons
+	rm --recursive --force $@
+	cp --recursive $< $@
+	rm --recursive --force $@/c2cgeoportal_commons/alembic
+	rm $@/tests.yaml.mako
+	touch $@
+
+.PHONY: docker-build-qgisserver
+docker-build-qgisserver: $(shell docker-required --path docker/qgisserver) docker/qgisserver/commons
+	docker build --tag=$(DOCKER_BASE)-qgisserver:$(MAJOR_VERSION) docker/qgisserver
+
+
 .PHONY: prepare-tests
 prepare-tests: \
 		geoportal/tests/functional/test.ini \

docker/qgisserver/.dockerignore

@@ -1,3 +1,4 @@
 *
-!accesscontrol.py
-!metadata.txt
+!geomapfish_plugin
+!commons
+!requirements.txt

docker/qgisserver/Dockerfile

@@ -1,4 +1,10 @@
-FROM camptocamp/qgis-server:3
+FROM camptocamp/qgis-server:latest
 LABEL maintainer Camptocamp "info@camptocamp.com"
 
-COPY * /var/www/plugins/geomapfish_qgisserver/
+COPY requirements.txt /tmp/
+RUN python3 -m pip install --requirement /tmp/requirements.txt
+
+COPY geomapfish_plugin/* /var/www/plugins/geomapfish_qgisserver/
+COPY commons /opt/c2cgeoportal_commons
+
+RUN python3 -m pip install --editable /opt/c2cgeoportal_commons

docker/qgisserver/accesscontrol.py → docker/qgisserver/geomapfish_plugin/accesscontrol.py

@@ -11,16 +11,21 @@
 
 from qgis.core import QgsMessageLog
 
+import json
 import os
 import traceback
 from shapely import ops
 import geoalchemy2
 import sqlalchemy
-import sqlahelper
 
 from qgis.server import QgsAccessControlFilter
 from qgis.core import QgsDataSourceUri
-import c2cgeoportal
+
+from sqlalchemy.orm import configure_mappers
+from sqlalchemy.orm import scoped_session
+from sqlalchemy.orm import sessionmaker
+
+from c2cgeoportal_commons.config import config
 
 
 class GMFException(Exception):
@@ -39,48 +44,43 @@ def __init__(self, server_iface):
         self.server_iface = server_iface
         self.area_cache = {}
 
-        try:
-            if "GEOMAPFISH_SCHEMA" not in os.environ:
-                raise GMFException("The environment variable 'GEOMAPFISH_SCHEMA' is not defined.")
-            if "GEOMAPFISH_SRID" not in os.environ:
-                raise GMFException("The environment variable 'GEOMAPFISH_SRID' is not defined.")
-            if "GEOMAPFISH_OGCSERVER" not in os.environ:
-                raise GMFException("The environment variable 'GEOMAPFISH_OGCSERVER' is not defined.")
-            if "GEOMAPFISH_SQLALCHEMYURL" not in os.environ:
-                raise GMFException("The environment variable 'GEOMAPFISH_SQLALCHEMYURL' is not defined.")
-
-            c2cgeoportal.schema = os.environ["GEOMAPFISH_SCHEMA"]
-            c2cgeoportal.srid = os.environ["GEOMAPFISH_SRID"]
-
-            sqlahelper.add_engine(sqlalchemy.create_engine(os.environ["GEOMAPFISH_SQLALCHEMYURL"]))
-
-            from c2cgeoportal.models import DBSession, LayerWMS, OGCServer
-
-            self.ogcserver = DBSession.query(OGCServer).filter(OGCServer.name == unicode(os.environ["GEOMAPFISH_OGCSERVER"])).one()
-
-            self.layers = {}
-            # TODO manage groups ...
-            for layer in DBSession.query(LayerWMS).filter(LayerWMS.ogc_server_id == self.ogcserver.id).all():
-                for name in layer.layer.split(','):
-                    if name not in self.layers:
-                        self.layers[name] = []
-                    self.layers[name].append(layer)
-
-            server_iface.registerAccessControl(self, 100)
-        except Exception as e:
-            QgsMessageLog.logMessage(traceback.format_tb(e))
-            QgsMessageLog.logMessage(str(e))
-            raise
+        if "GEOMAPFISH_OGCSERVER" not in os.environ:
+            raise GMFException("The environment variable 'GEOMAPFISH_OGCSERVER' is not defined.")
 
-    @staticmethod
-    def get_role():
-        from c2cgeoportal.models import DBSession, Role
+        self.srid = os.environ["GEOMAPFISH_SRID"]
+
+        config.init(os.environ.get('GEOMAPFISH_CONFIG', '/etc/qgisserver/geomapfish.yaml'))
+        self.config = config
+
+        from c2cgeoportal_commons.models.main import LayerWMS, OGCServer
+        configure_mappers()
+
+        engine = sqlalchemy.create_engine(config.get('sqlalchemy_slave.url'))
+        session_factory = sessionmaker()
+        session_factory.configure(bind=engine)
+        self.DBSession = scoped_session(session_factory)
+
+        self.ogcserver = self.DBSession.query(OGCServer) \
+            .filter(OGCServer.name == os.environ["GEOMAPFISH_OGCSERVER"]) \
+            .one()
+
+        self.layers = {}
+        # TODO manage groups ...
+        for layer in self.DBSession.query(LayerWMS).filter(LayerWMS.ogc_server_id == self.ogcserver.id).all():
+            for name in layer.layer.split(','):
+                if name not in self.layers:
+                    self.layers[name] = []
+                self.layers[name].append(layer)
+        QgsMessageLog.logMessage('[accesscontrol] layers: {}'.format(
+            json.dumps(self.layers, sort_keys=True, indent=4)
+        ))
 
-        # headers = self.serverInterface().requestHandler()
+        server_iface.registerAccessControl(self, int(os.environ.get("GEOMAPFISH_POSITION", 100)))
 
-        # return DBSession.query(User).get(headers.parameterMap()['USER_ID']).role
-        # return DBSession.query(Role).get(headers.parameterMap()['ROLE'])
-        return DBSession.query(Role).first()
+    def get_role(self):
+        from c2cgeoportal_commons.models.main import Role
+        parameters = self.serverInterface().requestHandler().parameterMap()
+        return self.DBSession.query(Role).get(parameters['ROLE_ID'])
 
     def get_restriction_areas(self, gmf_layers, rw=False, role=False):
         """
@@ -134,9 +134,9 @@ def layerFilterSubsetString(self, layer):  # NOQA
             if area is None:
                 return None
             area = "ST_GeomFromText('{}', {})".format(
-                area, c2cgeoportal.srid
+                area, self.srid
             )
-            if int(c2cgeoportal.srid) != layer.crs().postgisSrid():
+            if self.srid != layer.crs().postgisSrid():
                 area = "ST_transform({}, {})".format(
                     area, layer.crs().postgisSrid()
                 )
@@ -169,7 +169,7 @@ def layerFilterExpression(self, layer):  # NOQA
             # TODO cache the union
             # TODO verify the geometry
             return "intersects($geometry, transform(geom_from_wkt('{}'), 'EPSG:{}', 'EPSG:{}')".format(
-                area, c2cgeoportal.srid, layer.crs().projectionAcronym()
+                area, self.srid, layer.crs().projectionAcronym()
             )
         except Exception:
             QgsMessageLog.logMessage(traceback.format_exc())

docker/qgisserver/requirements.txt

@@ -0,0 +1,12 @@
+c2cgeoform==2.0.dev20180605  # commons
+c2c.template==2.1.0.dev1  # geoportal
+ColanderAlchemy==0.3.3  # commons
+colander==1.4  # commons, admin
+deform==2.0.5  # commons, admin
+GeoAlchemy2==0.4.2  # commons, geoportal
+geojson==2.3.0  # geoportal
+iso8601==0.1.11  # deform, by default the version 0.1.12 is installed and is incompatible with deform, rq.filter: <=0.1.11
+papyrus==2.3  # commons, geoportal
+pyramid==1.9.1
+SQLAlchemy==1.2.6
+transaction==2.2.1  # commons, geoportal

geoportal/c2cgeoportal_geoportal/scaffolds/create/+dot+gitignore_tmpl

@@ -18,6 +18,7 @@ __pycache__/
 /mapserver/*.map.tmpl
 /mapserver/tinyows.xml.mako
 /mapserver/tinyows.xml.tmpl
+/qgisserver/geomapfish.yaml
 /geoportal/jsbuild/app.cfg
 /geoportal/Dockerfile
 /geoportal/development.ini

geoportal/c2cgeoportal_geoportal/scaffolds/create/docker-compose.yaml.mako

@@ -29,10 +29,11 @@ ${service_defaults('print', 8080)}\
 ${service_defaults('mapserver', 8080)}\
 
 ##  qgisserver:
-##    image: camptocamp/qgis-server:latest
+##    image: camptocamp/geomapfish-qgisserver:2.3
+##    user: www-data
 ##    volumes_from:
 ##      - config:ro
-##${service_defaults('qgisserver', 80)}
+##${service_defaults('qgisserver', 8080)}
 
   tinyows:
     image: camptocamp/tinyows

geoportal/c2cgeoportal_geoportal/scaffolds/create/vars.yaml_tmpl

@@ -237,6 +237,11 @@ vars:
       Sincerely yours
       The GeoMapfish team
 
+  docker_services:
+    qgisserver:
+      environment:
+        GEOMAPFISH_OGCSERVER: <OGC server name>
+
   # Checker configuration
   checker:
     fulltextsearch:

geoportal/c2cgeoportal_geoportal/scaffolds/nondockercreate/+dot+gitignore_tmpl

@@ -32,6 +32,7 @@ __pycache__/
 /mapserver/*.map.tmpl.mako
 /mapserver/tinyows.xml
 /mapserver/tinyows.xml.tmpl.mako
+/qgisserver/geomapfish.yaml
 /tilegeneration/config.yaml
 /tilegeneration/config.yaml.tmpl.mako
 /geoportal/jsbuild/app.cfg

geoportal/c2cgeoportal_geoportal/scaffolds/update/CONST_Makefile_tmpl

@@ -395,7 +395,8 @@ git-attributes:
 	git --no-pager diff --check `git log --oneline | tail -1 | cut --fields=1 --delimiter=' '`
 
 YAML_FILES ?= $(filter-out ./tilegeneration/config.yaml ./geoportal/config.yaml \
-	./geoportal/alembic.yaml ./docker-compose-dev.yaml ./docker-compose.yaml, \
+	./qgisserver/geomapfish.yaml ./geoportal/alembic.yaml \
+	./docker-compose-dev.yaml ./docker-compose.yaml, \
 	$(shell find \
 		-name .build -prune -or \
 		-name cgxp -prune -or \
@@ -422,6 +423,7 @@ clean: template-clean
 		$(APP_OUTPUT_DIR)/ \
 		geoportal/$(PACKAGE)_geoportal/locale/$(PACKAGE)-*.pot \
 		geoportal/alembic.yaml \
+		qgisserver/geomapfish.yaml \
 		.UPGRADE* \
 		mapcache \
 		$(addprefix geoportal/$(PACKAGE)_geoportal/locale/, $(addsuffix /LC_MESSAGES/$(PACKAGE)_geoportal-$(L10N_CLIENT_POSTFIX).mo, $(LANGUAGES))) \
@@ -498,6 +500,13 @@ geoportal/config.yaml: /build/c2ctemplate-cache.json
 	mv /build/_config.yaml $@
 	touch $@
 
+qgisserver/geomapfish.yaml: /build/c2ctemplate-cache.json
+	$(PRERULE_CMD)
+	mkdir --parent $(dir $@)
+	c2c-template --cache /build/c2ctemplate-cache.json --get-config $@ \
+		sqlalchemy_slave.url schema schema_static srid
+	touch $@
+
 # server localisation
 
 .PRECIOUS: geoportal/$(PACKAGE)_geoportal/locale/$(PACKAGE)_geoportal-$(L10N_SERVER_POSTFIX).pot
@@ -690,7 +699,8 @@ docker-compose-build.yaml: /build/requirements.timestamp
 .PHONY: docker-build-config
 docker-build-config: $(shell docker-required --path .) \
 		$(PRINT_CONFIG_FILE) \
-		$(MAPCACHE_FILE)
+		$(MAPCACHE_FILE) \
+		qgisserver/geomapfish.yaml
 	docker build --tag=$(DOCKER_BASE)-config:$(DOCKER_TAG) .
 
 .PHONY: docker-build-geoportal

travis/publish-docker

@@ -2,7 +2,7 @@
 
 docker login --username ${DOCKER_USERNAME} --password ${DOCKER_PASSWORD}
 
-for IMAGE in geomapfish-build-dev geomapfish-commons geomapfish-build
+for IMAGE in geomapfish-build-dev geomapfish-commons geomapfish-build geomapfish-qgisserver
 do
     if [ "${TRAVIS_TAG}" != "" ]
     then

travis/short-make

@@ -15,4 +15,7 @@ if p.poll() is None:
     os.environ["DEBUG"] = "TRUE"
     subprocess.call(["make"] + sys.argv[1:])
     exit(2)
+if p.returncode != 0:
+    print("make call error:")
+    subprocess.call(["make"] + sys.argv[1:])
 exit(p.returncode)