Merge remote-tracking branch 'origin/2.3'

.github/ISSUE_TEMPLATE.md

@@ -3,7 +3,7 @@ To check before create a new issue:
 * I am using the latest minor version of the major version (`x.y.last`).
 * I create one issue per subject (do not put more than one thing in the same issue).
 
-For commiters:
+For committers:
 
 * I have assigned a green label.
 * I have assigned a milestone.

.gitignore

@@ -6,7 +6,6 @@
 /npm-packages
 /Dockerfile
 /commons/tests.yaml
-/commons/.build/
 /commons/c2cgeoportal_commons.egg-info/
 /geoportal/package.json
 /geoportal/tests/functional/test.ini
@@ -18,17 +17,12 @@
 /geoportal/c2cgeoportal_geoportal/scaffolds/create/geoportal/+package+_geoportal/locale/
 /geoportal/c2cgeoportal_geoportal/scaffolds/create/geoportal/+package+_geoportal/static-ngeo/images/
 /geoportal/c2cgeoportal_geoportal/scaffolds/create/geoportal/+package+_geoportal/static-ngeo/js/apps/
-/geoportal/c2cgeoportal_geoportal/scaffolds/create/geoportal/+package+_geoportal/static-ngeo/components/
 /geoportal/c2cgeoportal_geoportal/scaffolds/update/CONST_Makefile_tmpl.mk
 /geoportal/c2cgeoportal_geoportal/scaffolds/update/CONST_create_template/
+/geoportal/c2cgeoportal_geoportal/scaffolds/nondockercreate/geoportal/+package+_geoportal/static-ngeo/js/apps/
 /geoportal/c2cgeoportal_geoportal/scaffolds/nondockerupdate/CONST_create_template/
 /geoportal/c2cgeoportal_geoportal.egg-info/
 /admin/tests.ini
-/admin/.build/
-/admin/.pytest_cache/
-/admin/package-lock.json
-/admin/package.json
-/admin/node_modules/
 /admin/npm-packages
 /admin/c2cgeoportal_admin/locale/
 /admin/c2cgeoportal_admin.egg-info/

Jenkinsfile

@@ -39,7 +39,7 @@ dockerBuild {
                 sh 'docker run --name geomapfish-db --env=POSTGRES_USER=www-data --env=POSTGRES_PASSWORD=www-data --env=POSTGRES_DB=geomapfish --publish=5432:5432 --detach camptocamp/geomapfish-test-db'
                 sh 'travis/test-upgrade-convert.sh init ${HOME}/workspace'
             }
-            stage('Test') {
+            stage('Tests') {
                 checkout scm
                 parallel 'Lint and test c2cgeoportal': {
                     sh './docker-run travis/status.sh'
@@ -99,23 +99,24 @@ dockerBuild {
                     sh 'cat ${HOME}/workspace/testgeomapfish/testdb/*.sql'
                     sh 'cat ${HOME}/workspace/testgeomapfish/geoportal/config.yaml'
                     withCredentials([[
-                                             $class          : 'UsernamePasswordMultiBinding',
-                                             credentialsId   : 'dockerhub',
-                                             usernameVariable: 'USERNAME',
-                                             passwordVariable: 'PASSWORD'
-                                     ]]) {
+                        $class: 'UsernamePasswordMultiBinding',
+                        credentialsId: 'dockerhub',
+                        usernameVariable: 'USERNAME',
+                        passwordVariable: 'PASSWORD'
+                    ]]) {
                         try {
                             sh 'docker login -u "$USERNAME" -p "$PASSWORD"'
                             sh '(cd ${HOME}/workspace/testgeomapfish/; docker-compose up --force-recreate -d)'
                             sh '(cd ${HOME}/workspace/testgeomapfish/; docker-compose exec -T geoportal wait-for-db)'
                             sh './docker-run travis/waitwsgi http://`netstat --route --numeric|grep ^0.0.0.0|awk \'{print($2)}\'`:8080/'
                             for (path in [
-                                    'c2c/health_check',
-                                    'c2c/health_check?max_level=9',
-                                    'c2c/health_check?checks=check_collector',
-                                    'layers/test/values/type enum',
-                                    'admin/layertree',
-                                    'admin/layertree/children'
+                                'c2c/health_check',
+                                'c2c/health_check?max_level=9',
+                                'c2c/health_check?checks=check_collector',
+// TODO: activate gunicon logs to debug this test
+//                                'layers/test/values/type enum',
+                                'admin/layertree',
+                                'admin/layertree/children'
                             ]) {
                                 def start_lines = [:]
                                 ['db', 'external-db', 'print', 'mapserver', 'geoportal'].each { service ->

Makefile

@@ -64,8 +64,10 @@ ADMIN_SRC_FILES = $(shell ls -1 commons/c2cgeoportal_commons/models/*.py) \
 	$(shell find admin/c2cgeoportal_admin/templates/widgets -name "*.pt" -print)
 
 APPS += desktop mobile
+APPS_PACKAGE_PATH_NONDOCKER = geoportal/c2cgeoportal_geoportal/scaffolds/nondockercreate/geoportal/+package+_geoportal
+APPS_HTML_FILES = $(addprefix $(APPS_PACKAGE_PATH_NONDOCKER)/static-ngeo/js/apps/, $(addsuffix .html.ejs_tmpl, $(APPS)))
 APPS_PACKAGE_PATH = geoportal/c2cgeoportal_geoportal/scaffolds/create/geoportal/+package+_geoportal
-APPS_HTML_FILES = $(addprefix $(APPS_PACKAGE_PATH)/static-ngeo/js/apps/, $(addsuffix .html.ejs_tmpl, $(APPS)))
+APPS_HTML_FILES += $(addprefix $(APPS_PACKAGE_PATH)/static-ngeo/js/apps/, $(addsuffix .html.ejs_tmpl, $(APPS)))
 APPS_JS_FILES = $(addprefix $(APPS_PACKAGE_PATH)/static-ngeo/js/apps/Controller, $(addsuffix .js_tmpl, $(APPS)))
 APPS_ALT += desktop_alt mobile_alt oeedit oeview
 APPS_PACKAGE_PATH_ALT = geoportal/c2cgeoportal_geoportal/scaffolds/update/CONST_create_template/geoportal/+package+_geoportal/
@@ -251,12 +253,14 @@ quote:
 
 .PHONY: spell
 spell:
-	codespell --ignore-words=spell-ignore-words.txt geoportal/setup.py \
-		$(shell find commons/c2cgeoportal_commons -name '*.py' -print) \
-		$(shell find geoportal/c2cgeoportal_geoportal -name static -prune -or -name '*.py' -print) \
-		$(shell find admin/c2cgeoportal_admin -name '*.py' -print)
+	codespell --quiet-level=2 --check-filenames --ignore-words=spell-ignore-words.txt \
+		$(shell find -name node_modules -prune -or -name .build -prune -or -name .git -prune -or -name ngeo -prune \
+		-or -name '__pycache__' -prune -or -name _build -prune \
+		-or \( -type f -and -not -name '*.png' -and -not -name '*.mo' -and -not -name '*.po*' \
+		-and -not -name 'CONST_Makefile_tmpl' \) -print)
 
-YAML_FILES ?= $(shell find -name node_modules -prune -or -name .build -prune -or -name ngeo -prune -or -name functional -prune -or \( -name "*.yml" -or -name "*.yaml" \) -print)
+
+YAML_FILES ?= $(shell find -name ngeo -prune -or -name functional -prune -or \( -name "*.yml" -or -name "*.yaml" \) -print)
 .PHONY: yamllint
 yamllint:
 	yamllint --strict --config-file=yamllint.yaml -s $(YAML_FILES)
@@ -334,6 +338,11 @@ $(APPS_PACKAGE_PATH)/static-ngeo/js/apps/%.html.ejs_tmpl: ngeo/contribs/gmf/apps
 	mkdir --parent $(dir $@)
 	import-ngeo-apps --html $* $< $@
 
+$(APPS_PACKAGE_PATH_NONDOCKER)/static-ngeo/js/apps/%.html.ejs_tmpl: ngeo/contribs/gmf/apps/%/index.html.ejs
+	$(PRERULE_CMD)
+	mkdir --parent $(dir $@)
+	import-ngeo-apps --html --non-docker $* $< $@
+
 $(APPS_PACKAGE_PATH)/static-ngeo/js/apps/Controller%.js_tmpl: ngeo/contribs/gmf/apps/%/Controller.js
 	$(PRERULE_CMD)
 	mkdir --parent $(dir $@)
@@ -369,6 +378,7 @@ npm-packages: ngeo package.json
 		coveralls gaze jasmine-core jsdoc jsdom karma karma-chrome-launcher karma-coverage \
 		karma-jasmine karma-sourcemap-loader karma-webpack \
 		--src=ngeo/package.json --src=package.json --dst=$@
+	echo googshift eslint-plugin-googshift >> $@
 
 admin/npm-packages: ngeo package.json
 	$(PRERULE_CMD)

admin/c2cgeoportal_admin/__init__.py

@@ -56,8 +56,6 @@ def main(_, **settings):
     config.add_subscriber(add_renderer_globals, BeforeRender)
     config.add_subscriber(add_localizer, NewRequest)
 
-    config.add_tween('c2cgeoportal_admin.cache_tween.CacheTween')
-
     generate_mappers()
 
     health_check = HealthCheck(config)
@@ -93,9 +91,5 @@ def includeme(config: Configurator):
     config.include('pyramid_tm')
     config.add_translation_dirs('c2cgeoportal_admin:locale')
 
-    import c2cgeoportal_admin.cache_tween
-    c2cgeoportal_admin.cache_tween.route_prefix = config.route_prefix
-    config.add_tween('c2cgeoportal_admin.cache_tween.CacheTween')
-
     with PermissionSetter(config):
         config.scan()

admin/c2cgeoportal_admin/routes.py

@@ -9,7 +9,9 @@ def includeme(config):
         '{}:node_modules'.format(config.root_package.__name__))
     path = None
     for path_ in [
+        os.path.join(os.path.dirname(__file__), '..', '..', 'admin', 'node_modules'),
         os.path.join(os.path.dirname(__file__), '..', '..', 'node_modules'),
+        os.path.join(os.path.dirname(__file__), '..', 'admin', 'node_modules'),
         os.path.join(os.path.dirname(__file__), '..', 'node_modules'),
         '/usr/lib/node_modules/',
     ]:

admin/requirements.txt

@@ -1,4 +1,4 @@
-c2cgeoform==2.0.dev20180527
+c2cgeoform
 #c2cgeoportal-commons #  pip -> Could not find a version that satisfies the requirement c2cgeoportal-commons
 c2cwsgiutils
 colander

admin/c2cgeoportal_admin/cache_tween.py → commons/c2cgeoportal_commons/alembic/static/53d671b17b20_add_timezone_on_datetime_fields.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-# Copyright (c) 2011-2018, Camptocamp SA
+# Copyright (c) 2017-2018, Camptocamp SA
 # All rights reserved.
 
 # Redistribution and use in source and binary forms, with or without
@@ -27,19 +27,40 @@
 # of the authors and should not be interpreted as representing official policies,
 # either expressed or implied, of the FreeBSD Project.
 
-route_prefix = None
+"""Add timezone on datetime fields
 
+Revision ID: 53d671b17b20
+Revises: 1857owc78a07
+Create Date: 2018-05-29 08:44:17.675988
+"""
 
-class CacheTween:
+from alembic import op
+from c2cgeoportal_commons.config import config
 
-    def __init__(self, handler, registry):
-        del registry
-        self.handler = handler
+# revision identifiers, used by Alembic.
+revision = '53d671b17b20'
+down_revision = '1857owc78a07'
+branch_labels = None
+depends_on = None
 
-    def __call__(self, request):
-        # Never cache admin pages
-        response = self.handler(request)
-        if route_prefix is None or request.path_info.startswith('/' + route_prefix):
-            response.cache_control.no_cache = True
-            response.cache_control.max_age = 0
-        return response
+
+def upgrade():
+    staticschema = config['schema_static']
+
+    op.execute("""
+SET TIME ZONE 'UTC';
+ALTER TABLE {staticschema}.user ALTER COLUMN last_login TYPE timestamp with time zone;
+SET TIME ZONE LOCAL;
+ALTER TABLE {staticschema}.user ALTER COLUMN expire_on TYPE timestamp with time zone;
+""".format(staticschema=staticschema))
+
+
+def downgrade():
+    staticschema = config['schema_static']
+
+    op.execute("""
+SET TIME ZONE 'UTC';
+ALTER TABLE {staticschema}.user ALTER COLUMN last_login TYPE timestamp without time zone;
+SET TIME ZONE LOCAL;
+ALTER TABLE {staticschema}.user ALTER COLUMN expire_on TYPE timestamp without time zone;
+""".format(staticschema=staticschema))

commons/c2cgeoportal_commons/models/static.py

@@ -31,6 +31,7 @@
 import logging
 from datetime import datetime
 from hashlib import sha1
+import pytz
 from typing import Optional
 
 from sqlalchemy import Column
@@ -198,10 +199,10 @@ def validate_password(self, passwd: str) -> bool:
         return False
 
     def expired(self) -> bool:
-        return self.expire_on is not None and self.expire_on < datetime.utcnow()
+        return self.expire_on is not None and self.expire_on < datetime.now(pytz.utc)
 
     def update_last_login(self) -> None:
-        self.last_login = datetime.utcnow()
+        self.last_login = datetime.now(pytz.utc)
 
     def __unicode__(self) -> str:
         return self.username or ''  # pragma: no cover

doc/developer/debugging.rst

@@ -46,7 +46,7 @@ Sometime more information are available by using this command:
 You may also activate MapServer's debug mode and set environment variable of the MapServer container
 ``MS_DEBUGLEVEL`` to ``5`` (most verbose level, default is 0).
 
-`More informations <http://mapserver.org/optimization/debugging.html?highlight=debug#debug-levels>`_
+`More information <http://mapserver.org/optimization/debugging.html?highlight=debug#debug-levels>`_
 
 PostgreSQL
 ----------
@@ -67,19 +67,45 @@ Logs are available in the ``/var/log/postgresql/postgresql-9.*-main.log`` file.
 Makefile
 --------
 
-You can run `DEBUG=TRUE make ...` to have some debug message.
+To obtain additional debug messages, you can rebuild your project as follows:
+
+.. prompt:: bash
+
+   DEBUG=TRUE make ...
+
 Actually we display the running rule and why she is running (dependence update).
 
 Docker
 ------
 
-Edit a file in a running apache WSGI container
+Run gunicorn to reload on modifications of Python files
+.......................................................
+
+Add the following environment variable to the geoportal container:
+
+``GUNICORN_PARAMS="-b :80 --worker-class gthread --threads 1 --workers 1 --reload"``
+
+Do a graceful restart of the running geoportal container
+........................................................
 
 .. prompt:: bash
 
-   docker exec -ti <package>_wsgi_1 bash
-   vi ...
-   kill -s USR1 1  # graceful
+   docker-compose exec geoportal bash
+   kill -s HUP `ps aux|grep gunicorn|head --lines=1|awk '{print $2}'`  # graceful
+
+Mount c2cgeoportal in the container
+...................................
+
+Add in the ``docker-compose.yaml`` file, in the ``geoportal`` service the following lines:
+
+.. code:: yaml
+
+   services:
+     geoportal:
+       volumes:
+         - <c2cgeoportal_git_root>/geoportal/c2cgeoportal_commons:/opt/c2cgeoportal_geoportal/c2cgeoportal_commons
+         - <c2cgeoportal_git_root>/geoportal/c2cgeoportal_geoportal:/opt/c2cgeoportal_geoportal/c2cgeoportal_geoportal
+         - <c2cgeoportal_git_root>/geoportal/c2cgeoportal_admin:/opt/c2cgeoportal_geoportal/c2cgeoportal_admin
 
 
 Performance or network error

doc/developer/webservices.rst

@@ -240,18 +240,18 @@ Result HTTP code:
 * 200 Success: Success.
 * 400 Bad request: When something wrong.
 
-User informations
------------------
+User information
+----------------
 
-Used to get the user informations.
+Used to get the user information.
 
 URL: ``.../loginuser``
 
 Result HTTP code:
 
 * 200 Success: Success.
 
-Annoymous JSON result
+Anonymous JSON result
 ~~~~~~~~~~~~~~~~~~~~~
 
 .. code:: json

doc/integrator/fulltext_search.rst

@@ -37,7 +37,7 @@ Also make sure that the db user can ``SELECT`` in the ``tsearch`` table:
 
     sudo -u postgres psql -c 'GRANT SELECT ON TABLE <schema_name>.tsearch TO "<db_user>";' <db_name>
 
-with ``<db_user>``, and ``<db_name>`` substituded as appropriately.
+with ``<db_user>``, and ``<db_name>`` substituted as appropriately.
 
 Populate the full-text search table
 -----------------------------------

doc/integrator/security.rst

@@ -80,11 +80,11 @@ Services:
 - login
 - error
 
-Authorized referers
--------------------
+Authorized referrers
+--------------------
 
 To mitigate `CSRF <https://en.wikipedia.org/wiki/Cross-site_request_forgery>`_
-attacks, the server validates the referer against a list of authorized referers.
+attacks, the server validates the referer against a list of authorized referrers.
 
 By default, only the pages coming from the server are allowed. You can change
 that list by adding an ``authorized_referers`` list in your

docker/build/bin/eval-templates

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+
+find /app/ -name '*.tmpl' -print | while read file
+do
+    envsubst < ${file} > ${file%.tmpl}
+done
+
+exec "$@"

docker/build/bin/import-ngeo-apps

@@ -87,6 +87,7 @@ def main():
 
     parser.add_argument('--html', action="store_true", help="Import the html template")
     parser.add_argument('--js', action="store_true", help="Import the javascript controller")
+    parser.add_argument('--non-docker', action="store_false", dest="docker", help="Import for non Docker")
     parser.add_argument('interface', metavar='INTERFACE', help="The interface we import")
     parser.add_argument('src', metavar='SRC', help="The ngeo source file")
     parser.add_argument('dst', metavar='DST', help="The destination file")
@@ -116,7 +117,9 @@ def main():
             data = _sub(
                 re.escape('<script src="https://geomapfish-demo.camptocamp.com/2.') + '[0-9]' +
                 re.escape('/wsgi/dynamic.js?interface={}"></script>'.format(args.interface)),
-                '<script src="dynamic.js?interface={interface}"></script>'.format(interface=args.interface),
+                '<script src="{entry_point}dynamic.js?interface={interface}"></script>'.format(
+                    entry_point='${htmlWebpackPlugin.options.vars.entry_point}' if args.docker else '',
+                    interface=args.interface),
                 data
             )