Use pre-built docker images to speed up CI (#432)

dask · May 23, 2020 · b2a5ac8 · b2a5ac8
1 parent bb89969
commit b2a5ac8
Show file tree

Hide file tree

Showing 9 changed files with 25 additions and 24 deletions.
diff --git a/ci/pbs.sh b/ci/pbs.sh
@@ -6,6 +6,7 @@ function jobqueue_before_install {
 
     # start pbs cluster
     cd ./ci/pbs
+    docker-compose pull
     ./start-pbs.sh
     cd -
 

diff --git a/ci/pbs/docker-compose.yml b/ci/pbs/docker-compose.yml
@@ -3,6 +3,7 @@ version: "2"
 services:
 
   master:
+    image: daskdev/dask-jobqueue:pbs
     build: .
     container_name: pbs_master
     hostname: pbs_master
@@ -11,6 +12,7 @@ services:
     command: bash /run-master.sh
 
   slave_one:
+    image: daskdev/dask-jobqueue:pbs
     build: .
     container_name: pbs_slave_1
     hostname: pbs_slave_1
@@ -26,6 +28,7 @@ services:
       - master
 
   slave_two:
+    image: daskdev/dask-jobqueue:pbs
     build: .
     container_name: pbs_slave_2
     hostname: pbs_slave_2
@@ -39,4 +42,3 @@ services:
       - PBS_MASTER=pbs_master
     depends_on:
       - master
-
diff --git a/ci/sge.sh b/ci/sge.sh
@@ -3,8 +3,10 @@
 function jobqueue_before_install {
     docker version
     docker-compose version
+
     # start sge cluster
     cd ./ci/sge
+    docker-compose pull
     ./start-sge.sh
     cd -
 

diff --git a/ci/sge/Dockerfile-master → ci/sge/Dockerfile b/ci/sge/Dockerfile-master → ci/sge/Dockerfile
@@ -1,4 +1,4 @@
-FROM ubuntu:14.04
+FROM ubuntu:14.04 as base
 
 ENV LANG C.UTF-8
 
@@ -14,6 +14,11 @@ RUN conda install -c conda-forge python=$PYTHON_VERSION dask distributed pytest
 
 COPY ./*.sh /
 COPY ./*.txt /
+
+FROM base as slave
+RUN bash ./setup-slave.sh
+
+FROM base as master
 RUN bash ./setup-master.sh
 
 # expose ports

diff --git a/ci/sge/Dockerfile-slave b/ci/sge/Dockerfile-slave
diff --git a/ci/sge/docker-compose.yml b/ci/sge/docker-compose.yml
@@ -1,11 +1,12 @@
-version: "2.1"
+version: "3.4"
 
 services:
 
   master:
+    image: daskdev/dask-jobqueue:sge
     build:
       context: .
-      dockerfile: Dockerfile-master
+      target: master
       args:
         PYTHON_VERSION: ${TRAVIS_PYTHON_VERSION:-3.6}
     container_name: sge_master
@@ -16,9 +17,10 @@ services:
     command: bash /run-master.sh
 
   slave-one:
+    image: daskdev/dask-jobqueue:sge-slave
     build:
       context: .
-      dockerfile: Dockerfile-slave
+      target: slave
       args:
         PYTHON_VERSION: ${TRAVIS_PYTHON_VERSION:-3.6}
     container_name: slave_one
@@ -33,9 +35,10 @@ services:
       - master
 
   slave-two:
+    image: daskdev/dask-jobqueue:sge-slave
     build:
       context: .
-      dockerfile: Dockerfile-slave
+      target: slave
       args:
         PYTHON_VERSION: ${TRAVIS_PYTHON_VERSION:-3.6}
     container_name: slave_two

diff --git a/ci/slurm.sh b/ci/slurm.sh
@@ -6,6 +6,7 @@ function jobqueue_before_install {
 
     # start slurm cluster
     cd ./ci/slurm
+    docker-compose pull
     ./start-slurm.sh
     cd -
 

diff --git a/ci/slurm/docker-compose.yml b/ci/slurm/docker-compose.yml
@@ -16,6 +16,7 @@ services:
       common-network:
 
   slurmdbd:
+    image: daskdev/dask-jobqueue:slurm
     build: .
     command: ["slurmdbd"]
     container_name: slurmdbd
@@ -32,6 +33,7 @@ services:
       common-network:
 
   slurmctld:
+    image: daskdev/dask-jobqueue:slurm
     build: .
     command: ["slurmctld"]
     container_name: slurmctld
@@ -53,6 +55,7 @@ services:
       - NET_ADMIN
 
   c1:
+    image: daskdev/dask-jobqueue:slurm
     build: .
     command: ["slurmd"]
     hostname: c1
@@ -73,6 +76,7 @@ services:
       - NET_ADMIN
 
   c2:
+    image: daskdev/dask-jobqueue:slurm
     build: .
     command: ["slurmd"]
     hostname: c2

diff --git a/ci/slurm/start-slurm.sh b/ci/slurm/start-slurm.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-docker-compose up --build -d
+docker-compose up -d
 
 while [ `./register_cluster.sh 2>&1 | grep "sacctmgr: error" | wc -l` -ne 0 ]
   do