Merge pull request #4852 from jwhitlock/node-8-1468491

bug 1468491: Update node 8, dependencies
mdn · Jul 26, 2018 · b1e05e1 · b1e05e1
2 parents 4a3f945 + c7cc405
commit b1e05e1
Show file tree

Hide file tree

Showing 22 changed files with 8,491 additions and 5,969 deletions.
diff --git a/.env-dist.dev b/.env-dist.dev
@@ -3,7 +3,7 @@
 # User ID to use in Docker containers
 # Linux users should set this value to avoid permissions issues. Use "id" or
 # "echo $UID" to determine the user ID.
-# MacOS and Windows users can use the default of 33 (www-data), and the Docker
+# MacOS and Windows users can use the default of 1000 (kuma), and the Docker
 #  file system layer will translate to your local user permissions.
 #UID=1000
 

diff --git a/.travis.yml b/.travis.yml
@@ -31,8 +31,8 @@ matrix:
           env:
             TOXENV=pytest
             CREATE_DB=kuma
-            INSTALL_PIPELINE=1
             INSTALL_ELASTICSEARCH=1
+            INSTALL_NPM_TOOLS=1
         - python: "2.7"
           env:
             TOXENV=flake8
@@ -43,31 +43,32 @@ matrix:
           env:
             TOXENV=locales
             CREATE_DB=kuma
-            INSTALL_PIPELINE=1
+            INSTALL_NPM_TOOLS=1
         - python: "2.7"
           env:
             TOXENV=docker
             INSTALL_DOCKER_COMPOSE=1
             UID=0
         - python: "2.7"
           env:
-            TOXENV=stylelint
+            TOXENV=assetlint
+            INSTALL_NPM_TOOLS=1
         - python: "3.6"
           env:
             TOXENV=pytest
             CREATE_DB=kuma
-            INSTALL_PIPELINE=1
             INSTALL_ELASTICSEARCH=1
+            INSTALL_NPM_TOOLS=1
     allow_failures:
         - python: "3.6"
           env:
             TOXENV=pytest
             CREATE_DB=kuma
-            INSTALL_PIPELINE=1
             INSTALL_ELASTICSEARCH=1
+            INSTALL_NPM_TOOLS=1
 install:
-    - nvm install 6
-    - nvm use 6
+    - nvm install 8
+    - nvm use 8
     - scripts/travis-install
     - pip install -U pip
     - pip install -r requirements/travis.txt

diff --git a/Makefile b/Makefile
@@ -148,5 +148,10 @@ shell_plus: up
 lint:
 	flake8 kuma docs tests
 
+npmrefresh:
+	cd /tools
+	echo '{"lockfileVersion": 1}' > package-lock.json
+	npm install
+
 # Those tasks don't have file targets
-.PHONY: test coveragetest locust clean locale install compilejsi18n collectstatic localetest localeextract localecompile localerefresh
+.PHONY: test coveragetest locust clean locale install compilejsi18n collectstatic localetest localeextract localecompile localerefresh npmrefresh
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -3,7 +3,7 @@ services:
   worker: &worker
     image: quay.io/mozmar/kuma_base
     command: ./manage.py celery worker --loglevel=INFO --events --beat --autoreload --concurrency=4 -Q mdn_purgeable,mdn_search,mdn_emails,mdn_wiki,celery
-    user: ${UID:-33}
+    user: ${UID:-1000}
     volumes:
       - ./:/app:z
     depends_on:

diff --git a/docker/images/kuma/Dockerfile b/docker/images/kuma/Dockerfile
@@ -2,16 +2,10 @@ FROM quay.io/mozmar/kuma_base:latest
 
 ARG REVISION_HASH
 # Make the git commit hash permanently available within this image.
-ENV REVISION_HASH $REVISION_HASH
+ENV REVISION_HASH=$REVISION_HASH \
+    DJANGO_SETTINGS_MODULE=kuma.settings.prod
 
-COPY . /app
-# the following is needed until the --user flag is added to COPY
-# see https://github.com/moby/moby/issues/30110
-USER root
-RUN chown -R kuma /app
-USER kuma
-
-ENV DJANGO_SETTINGS_MODULE=kuma.settings.prod
+COPY --chown=kuma:kuma . /app
 
 # Temporarily enable candidate languages so assets are built for all
 # environments, but still defaults to disabled in production.

diff --git a/docker/images/kuma_base/Dockerfile b/docker/images/kuma_base/Dockerfile
@@ -1,7 +1,7 @@
 FROM python:2.7-slim
 
 # Set the environment variables
-ENV NODE_VERSION=6.14.2 \
+ENV NODE_VERSION=8.11.3 \
     # extra python env
     PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
@@ -21,7 +21,6 @@ RUN set -x \
     && apt-get update \
     && apt-get install -y --no-install-recommends \
         curl \
-        xz-utils \
         gpg \
         dirmngr \
         libsasl2-modules \
@@ -39,11 +38,15 @@ RUN set -x \
     && rm -rf /var/lib/apt/lists/*
 
 # ----------------------------------------------------------------------------
-# add node.js 6.x, copied from:
-#     https://github.com/nodejs/docker-node/blob/master/6/stretch/Dockerfile
-# but with package updates and version definitions moved above
+# add node.js 8.x, copied from:
+#     https://github.com/nodejs/docker-node/blob/master/8/stretch/Dockerfile
+# but with package updates and version definitions moved above, and the node
+# user gets uid/gid 1001 rather than 1000.
 # ----------------------------------------------------------------------------
 
+RUN groupadd --gid 1001 node \
+  && useradd --uid 1001 --gid node --shell /bin/bash --create-home node
+
 RUN set -ex \
   && for key in \
     94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
@@ -54,6 +57,7 @@ RUN set -ex \
     B9AE9905FFD7803F25714661B63B535A4C206CA9 \
     56730D5401028683275BD23C23EFEFE93C4CFFFE \
     77984A986EBC2AA786BC0F66B01FBB92821C587A \
+    8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
   ; do \
     gpg --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \
     gpg --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \
@@ -79,28 +83,37 @@ RUN ARCH=  && dpkgArch="$(dpkg --print-architecture)" \
     && ln -s /usr/local/bin/node /usr/local/bin/nodejs
 # ----------------------------------------------------------------------------
 
-# add non-priviledged user
-RUN adduser --uid 1000 --disabled-password --gecos '' --no-create-home kuma
+# add non-privileged user
+RUN useradd --uid 1000 --shell /bin/bash --create-home kuma \
+    && mkdir -p app \
+    && chown kuma:kuma /app \
+    && chmod 775 /app
 
+# install Python libraries
 WORKDIR /app
-EXPOSE 8000
-
-RUN npm install -g \
-        fibers@1.0.15 \
-        csslint@0.10.0 \
-        jshint@2.7.0 \
-        node-sass@4.3.0 \
-        uglify-js@2.4.13 \
-        clean-css@3.4.23 \
-        stylelint@7.10.1
-
-COPY ./requirements /app/requirements
+COPY --chown=kuma:kuma ./requirements /app/requirements
 RUN pip install --no-cache-dir -r requirements/dev.txt
 
-# Import issue with configparser 3.5.0
+# fix import issue with configparser 3.5.0
 # https://bitbucket.org/ambv/configparser/issues/20/importwarning-missing-__init__py
 RUN touch /usr/local/lib/python2.7/site-packages/backports/__init__.py
 
+# install Node.js tools
+# config files are symlinks to make updating easier
+COPY --chown=kuma:kuma ./package.json ./package-lock.json /app/
+RUN mkdir /tools \
+    && chown kuma:kuma /tools \
+    && chmod 775 /tools
+WORKDIR /tools
 USER kuma
+RUN ln -s /app/package.json /tools \
+    && ln -s /app/package-lock.json /tools \
+    && npm install --production
+USER root
+RUN find /tools/node_modules/.bin/ -executable -type f -o -type l -exec ln -s {} /usr/local/bin/ \;
 
+# setup default run parameters
+USER kuma
+WORKDIR /app
+EXPOSE 8000
 CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--timeout=120", "--worker-class=meinheld.gmeinheld.MeinheldWorker", "kuma.wsgi:application"]
diff --git a/docker/images/kuma_base/Dockerfile-py3 b/docker/images/kuma_base/Dockerfile-py3
@@ -1,7 +1,7 @@
 FROM python:3.6-slim
 
 # Set the environment variables
-ENV NODE_VERSION=6.14.2 \
+ENV NODE_VERSION=8.11.3 \
     # extra python env
     PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
@@ -21,7 +21,6 @@ RUN set -x \
     && apt-get update \
     && apt-get install -y --no-install-recommends \
         curl \
-        xz-utils \
         gpg \
         dirmngr \
         libsasl2-modules \
@@ -39,11 +38,15 @@ RUN set -x \
     && rm -rf /var/lib/apt/lists/*
 
 # ----------------------------------------------------------------------------
-# add node.js 6.x, copied from:
-#     https://github.com/nodejs/docker-node/blob/master/6/stretch/Dockerfile
-# but with package updates and version definitions moved above
+# add node.js 8.x, copied from:
+#     https://github.com/nodejs/docker-node/blob/master/8/stretch/Dockerfile
+# but with package updates and version definitions moved above, and the node
+# user gets uid/gid 1001 rather than 1000.
 # ----------------------------------------------------------------------------
 
+RUN groupadd --gid 1001 node \
+  && useradd --uid 1001 --gid node --shell /bin/bash --create-home node
+
 RUN set -ex \
   && for key in \
     94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
@@ -54,6 +57,7 @@ RUN set -ex \
     B9AE9905FFD7803F25714661B63B535A4C206CA9 \
     56730D5401028683275BD23C23EFEFE93C4CFFFE \
     77984A986EBC2AA786BC0F66B01FBB92821C587A \
+    8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
   ; do \
     gpg --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \
     gpg --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \
@@ -79,24 +83,33 @@ RUN ARCH=  && dpkgArch="$(dpkg --print-architecture)" \
     && ln -s /usr/local/bin/node /usr/local/bin/nodejs
 # ----------------------------------------------------------------------------
 
-# add non-priviledged user
-RUN adduser --uid 1000 --disabled-password --gecos '' --no-create-home kuma
+# add non-privileged user
+RUN useradd --uid 1000 --shell /bin/bash --create-home kuma \
+    && mkdir -p app \
+    && chown kuma:kuma /app \
+    && chmod 775 /app
 
+# install Python libraries
 WORKDIR /app
-EXPOSE 8000
-
-RUN npm install -g \
-        fibers@1.0.15 \
-        csslint@0.10.0 \
-        jshint@2.7.0 \
-        node-sass@4.3.0 \
-        uglify-js@2.4.13 \
-        clean-css@3.4.23 \
-        stylelint@7.10.1
-
-COPY ./requirements /app/requirements
+COPY --chown=kuma:kuma ./requirements /app/requirements
 RUN pip install --no-cache-dir -r requirements/dev.txt
 
+# install Node.js tools
+# config files are symlinks to make updating easier
+COPY --chown=kuma:kuma ./package.json ./package-lock.json /app/
+RUN mkdir /tools \
+    && chown kuma:kuma /tools \
+    && chmod 775 /tools
+WORKDIR /tools
 USER kuma
+RUN ln -s /app/package.json /tools \
+    && ln -s /app/package-lock.json /tools \
+    && npm install --production
+USER root
+RUN find /tools/node_modules/.bin/ -executable -type f -o -type l -exec ln -s {} /usr/local/bin/ \;
 
+# setup default run parameters
+USER kuma
+WORKDIR /app
+EXPOSE 8000
 CMD ["gunicorn", "--bind", "0.0.0.0:8000", "--timeout=120", "--worker-class=meinheld.gmeinheld.MeinheldWorker", "kuma.wsgi:application"]