Skip to content

Commit

Permalink
Fix #14: updated to next-generation deployment with ansible
Browse files Browse the repository at this point in the history
  • Loading branch information
cehbrecht authored and Zeitsperre committed Jan 5, 2021
1 parent 24c3ee2 commit fa89852
Show file tree
Hide file tree
Showing 22 changed files with 266 additions and 456 deletions.
3 changes: 3 additions & 0 deletions .dockerignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
.git
docs/
tests/
6 changes: 4 additions & 2 deletions cookiecutter.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,10 @@
"add_pyup_badge": "n",
"command_line_interface": ["Click", "Argparse", "No command-line interface"],
"create_author_file": "y",
"open_source_license": ["MIT license", "BSD license", "ISC license", "Apache Software License 2.0", "GNU General Public License v3", "Not open source"],
"open_source_license": ["Apache Software License 2.0", "MIT license", "BSD license", "ISC license", "GNU General Public License v3", "Not open source"],
"port_digits": "94",
"http_port": "80{{ cookiecutter.port_digits }}",
"https_port": "280{{ cookiecutter.port_digits }}"
"https_port": "280{{ cookiecutter.port_digits }}",
"_copy_without_render": [
"{{cookiecutter.project_slug}}/templates/*.cfg",
}
60 changes: 23 additions & 37 deletions {{cookiecutter.project_slug}}/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,47 +1,33 @@
# vim:set ft=dockerfile:
FROM birdhouse/bird-base:latest
FROM continuumio/miniconda3
MAINTAINER https://github.com/{{ cookiecutter.github_username }}/{{ cookiecutter.project_slug }}
LABEL Description="{{ cookiecutter.project_slug }} WPS" Vendor="Birdhouse" Version="{{ cookiecutter.version }}"

LABEL Description="{{ cookiecutter.project_slug }} WPS" Vendor="Birdhouse"
# Update Debian system
RUN apt-get update && apt-get install -y \
build-essential \
&& rm -rf /var/lib/apt/lists/*

# Configure hostname and ports for services
ENV HTTP_PORT 5000
ENV OUTPUT_PORT 8000
ENV HOSTNAME localhost
# Update conda
RUN conda update -n base conda

# Set current home
ENV HOME /root
# Copy WPS project
COPY . /opt/wps

# Copy application sources
COPY . /opt/birdhouse/src/{{ cookiecutter.project_slug }}
WORKDIR /opt/wps

# cd into application
WORKDIR /opt/birdhouse/src/{{ cookiecutter.project_slug }}
# Create conda environment
RUN conda env create -n wps -f environment.yml

# Provide custom.cfg with settings for docker image
COPY .docker.cfg custom.cfg
# Install WPS
RUN ["/bin/bash", "-c", "source activate wps && python setup.py develop"]

# Install system dependencies
RUN bash bootstrap.sh -i
# Start WPS service on port {{ cookiecutter.http_port }} on 0.0.0.0
EXPOSE {{ cookiecutter.http_port }}
ENTRYPOINT ["/bin/bash", "-c"]
CMD ["source activate wps && exec emu --config /opt/wps/etc/demo.cfg"]

# Set conda enviroment
ENV ANACONDA_HOME /opt/conda
ENV CONDA_ENVS_DIR /opt/conda/envs

# Run install and fix permissions
RUN make clean install && chmod 755 /opt/birdhouse/etc && chmod 755 /opt/birdhouse/var/run

# Volume for data, cache, logfiles, ...
VOLUME /opt/birdhouse/var/lib
VOLUME /opt/birdhouse/var/log
# Volume for configs
VOLUME /opt/birdhouse/etc

# Ports used in birdhouse
EXPOSE $HTTP_PORT $OUTPUT_PORT

# Start supervisor in foreground
ENV DAEMON_OPTS --nodaemon

# Start service ...
CMD ["make", "update-config", "start"]
# docker build -t {{ cookiecutter.github_username }}/{{ cookiecutter.project_slug }} .
# docker run -p {{ cookiecutter.http_port }}:{{ cookiecutter.http_port }} {{ cookiecutter.github_username }}/{{ cookiecutter.project_slug }}
# http://localhost:{{ cookiecutter.http_port }}/wps?request=GetCapabilities&service=WPS
# http://localhost:{{ cookiecutter.http_port }}/wps?request=DescribeProcess&service=WPS&identifier=all&version=1.0.0
103 changes: 34 additions & 69 deletions {{cookiecutter.project_slug}}/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,6 @@ BUILDOUT_VERSION := 2.11
# Anaconda
ANACONDA_HOME ?= $(HOME)/anaconda
CONDA_ENV ?= $(APP_NAME)
CONDA_ENVS_DIR ?= $(HOME)/.conda/envs
CONDA_ENV_PATH := $(CONDA_ENVS_DIR)/$(CONDA_ENV)
CONDA_PINNED := $(APP_ROOT)/requirements/conda_pinned

# Configuration used by update-config
HOSTNAME ?= localhost
Expand All @@ -32,25 +29,22 @@ OUTPUT_PORT ?= 8090
# choose anaconda installer depending on your OS
ANACONDA_URL = https://repo.continuum.io/miniconda
ifeq "$(OS_NAME)" "Linux"
FN := Miniconda2-latest-Linux-x86_64.sh
FN := Miniconda3-latest-Linux-x86_64.sh
else ifeq "$(OS_NAME)" "Darwin"
FN := Miniconda2-latest-MacOSX-x86_64.sh
FN := Miniconda3-latest-MacOSX-x86_64.sh
else
FN := unknown
endif

# Buildout files and folders
DOWNLOAD_CACHE := $(APP_ROOT)/downloads
BUILDOUT_FILES := parts eggs develop-eggs bin .installed.cfg .mr.developer.cfg *.egg-info bootstrap-buildout.py *.bak.* $(DOWNLOAD_CACHE)
TEMP_FILES := *.egg-info *.log *.sqlite

# Docker
DOCKER_IMAGE := birdhouse/$(APP_NAME)
DOCKER_CONTAINER := $(APP_NAME)

# end of configuration

.PHONY: clean clean-test clean-pyc clean-build docs help
.DEFAULT_GOAL := help
.DEFAULT_GOAL := all

define BROWSER_PYSCRIPT
import os, webbrowser, sys
Expand Down Expand Up @@ -207,73 +201,42 @@ anaconda:
@echo "Installing Anaconda ..."
@test -d $(ANACONDA_HOME) || curl $(ANACONDA_URL)/$(FN) --silent --insecure --output "$(DOWNLOAD_CACHE)/$(FN)"
@test -d $(ANACONDA_HOME) || bash "$(DOWNLOAD_CACHE)/$(FN)" -b -p $(ANACONDA_HOME)
@echo "Add '$(ANACONDA_HOME)/bin' to your PATH variable in '.bashrc'."

.PHONY: conda_config
conda_config: anaconda
@echo "Update ~/.condarc"
@-"$(ANACONDA_HOME)/bin/conda" install -y conda=$(CONDA_VERSION) requests
@"$(ANACONDA_HOME)/bin/conda" config --add envs_dirs $(CONDA_ENVS_DIR)
@"$(ANACONDA_HOME)/bin/conda" config --set ssl_verify true
@"$(ANACONDA_HOME)/bin/conda" config --set use_pip true
@"$(ANACONDA_HOME)/bin/conda" config --set channel_priority true
@"$(ANACONDA_HOME)/bin/conda" config --set auto_update_conda false
@"$(ANACONDA_HOME)/bin/conda" config --add channels defaults
@"$(ANACONDA_HOME)/bin/conda" config --append channels birdhouse
@"$(ANACONDA_HOME)/bin/conda" config --append channels conda-forge
@echo "Please add '$(ANACONDA_HOME)/bin' to your PATH variable in '.bashrc'."

.PHONY: conda_env
conda_env: anaconda conda_config
@echo "Update conda environment $(CONDA_ENV) ..."
@test -d $(CONDA_ENV_PATH) || "$(ANACONDA_HOME)/bin/conda" env create -n $(CONDA_ENV) -f environment.yml
"$(ANACONDA_HOME)/bin/conda" install -y -n $(CONDA_ENV) setuptools=$(SETUPTOOLS_VERSION)

.PHONY: conda_pinned
conda_pinned: conda_env
@echo "Update pinned conda packages ..."
@-test -d $(CONDA_ENV_PATH) && test -f $(CONDA_PINNED) && cp -f "$(CONDA_PINNED)" "$(CONDA_ENV_PATH)/conda-meta/pinned"

.PHONY: export
export:
@echo "Exporting conda enviroment ..."
@test -d $(CONDA_ENV_PATH) && "$(ANACONDA_HOME)/bin/conda" env export -n $(CONDA_ENV) -f environment.yml
conda_env: anaconda
@echo "Updating conda environment $(CONDA_ENV) ..."
"$(ANACONDA_HOME)/bin/conda" env update -n $(CONDA_ENV) -f environment.yml

## Build targets

.PHONY: bootstrap
bootstrap: init conda_env conda_pinned bootstrap-buildout.py
@echo "Bootstrap buildout ..."
@test -f bin/buildout || bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);python bootstrap-buildout.py -c custom.cfg --allow-site-packages --setuptools-version=$(SETUPTOOLS_VERSION) --buildout-version=$(BUILDOUT_VERSION)"
bootstrap: conda_env bootstrap_dev
@echo "Bootstrap ..."

.PHONY: sysinstall
sysinstall:
@echo "\nInstalling system packages for bootstrap ..."
@bash bootstrap.sh -i
@echo "\nInstalling system packages for your application ..."
@-test -f requirements.sh && bash requirements.sh
.PHONY: bootstrap_dev
bootstrap_dev:
@echo "Installing development requirements for tests and docs ..."
@-bash -c "$(ANACONDA_HOME)/bin/conda install -y -n $(CONDA_ENV) pytest flake8 sphinx"
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && pip install -r requirements_dev.txt"

.PHONY: install
install: bootstrap
@echo "Installing application with buildout ..."
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);bin/buildout buildout:anaconda-home=$(ANACONDA_HOME) -c custom.cfg"
@echo "Installing application..."
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && python setup.py develop"
@echo "\nStart service with \`make start'"

.PHONY: update
update:
@echo "Update application config with buildout (offline mode) ..."
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);bin/buildout buildout:anaconda-home=$(ANACONDA_HOME) -o -c custom.cfg"

.PHONY: update-config
update-config:
@echo "Update application config with buildout (offline mode) and environment variables..."
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);bin/buildout buildout:anaconda-home=$(ANACONDA_HOME) settings:hostname=$(HOSTNAME) settings:output-port=$(OUTPUT_PORT) settings:http-port=$(HTTP_PORT) -o -c custom.cfg"
.PHONY: start
start:
@echo "Starting application ..."
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && $(APP_NAME) -a -d"

.PHONY: clean
clean: srcclean envclean
@echo "Cleaning buildout files ..."
@-for i in $(BUILDOUT_FILES); do \
test -e $$i && rm -v -rf $$i; \
done
@echo "Cleaning generated files ..."
@-for i in $(TEMP_FILES); do \
test -e $$i && rm -v -rf $$i; \
done

.PHONY: envclean
envclean: stop
Expand All @@ -286,10 +249,10 @@ srcclean:
@-find $(APP_ROOT) -type f -name "*.pyc" -print | xargs rm

.PHONY: distclean
distclean: backup clean
@echo "Cleaning distribution ..."
distclean: clean
@echo "Cleaning ..."
@git diff --quiet HEAD || echo "There are uncommited changes! Not doing 'git clean' ..."
@-git clean -dfx -e *.bak -e custom.cfg -e Makefile.config
@-git clean -dfx -e *.bak -e custom.cfg

.PHONY: passwd
passwd: custom.cfg
Expand All @@ -302,22 +265,24 @@ passwd: custom.cfg
.PHONY: test
test:
@echo "Running tests (skip slow and online tests) ..."
bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV); bin/py.test -v -m 'not slow and not online'"
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);pytest -v -m 'not slow and not online'"

.PHONY: testall
testall:
@echo "Running all tests (including slow and online tests) ..."
bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV); bin/py.test -v"
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && pytest -v"

.PHONY: pep8
pep8:
@echo "Running pep8 code style checks ..."
$(CONDA_ENV_PATH)/bin/flake8
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV) && flake8"

## Sphinx targets

.PHONY: docs
docs:
@echo "Generating docs with Sphinx ..."
$(MAKE) -C $@ clean html
@-bash -c "source $(ANACONDA_HOME)/bin/activate $(CONDA_ENV);$(MAKE) -C $@ clean html"
@echo "open your browser: firefox docs/build/html/index.html"

.PHONY: linkcheck
Expand Down
3 changes: 0 additions & 3 deletions {{cookiecutter.project_slug}}/Makefile.config.example

This file was deleted.

67 changes: 0 additions & 67 deletions {{cookiecutter.project_slug}}/bootstrap.sh

This file was deleted.

Loading

0 comments on commit fa89852

Please sign in to comment.