Makefile.cluster

# -*- mode: makefile -*-
# saner makefile
MAKEFLAGS += --no-builtin-rules
.SUFFIXES:

include Makevars

SSH_IDENTITY_FILE := docker-image/id_rsa
SSH_CONTROL_FILE_PREFIX := $(CURDIR)/ssh-controlmaster-
SSH_HOST_PORT := 2222
SSH := ssh -o ControlPath=$(SSH_CONTROL_FILE_PREFIX)%r@%h:%p -F none -o LogLevel=ERROR -o "UserKnownHostsFile=/dev/null" -o "StrictHostKeyChecking=no" -p $(SSH_HOST_PORT) -l r -i $(SSH_IDENTITY_FILE) $(SSH_EXTRA)
SSH_NUM_SLOTS := 64

# all hosts
HOSTS := prl2 prl3 prl4
# slaves (excluding the one from which we run this make file)
SLAVES = $(filter-out $(HOSTNAME),$(HOSTS))

HOSTNAME := $(shell hostname)
WORKER_NAME := evalr-worker-$(HOSTNAME)

THIS_MAKEFILE := $(lastword $(MAKEFILE_LIST))
THIS_MAKE := $(MAKE) -C $(CURDIR) -f $(THIS_MAKEFILE)

.PHONY: image
image:
	$(MAKE) -f Makefile docker-image

.PHONY: .r-dyntrace-docker-image
.r-dyntrace-docker-image:
	-docker rm -f $(WORKER_NAME)
	-docker rmi -f $(DOCKER_IMAGE_NAME) $(DOCKER_R_DYNTRACE_IMAGE_NAME)
	$(MAKE) -C $(CURDIR) -f $(THIS_MAKEFILE) image

.PHONY: .node-setup
.node-setup: image
	@if [ ! -d $(LIBRARY_DIR) ]; then echo "Library dir: '$(LIBRARY_DIR)' does not exist"; exit 1; fi
	@if [ ! -d $(CRAN_DIR) ]; then echo "CRAN dir '$(CRAN_DIR)'' does not exist"; exit 1; fi
	rm -f $(SSH_CONTROL_FILE_PREFIX)*$(HOSTNAME)*
	-docker rm -f $(WORKER_NAME) 2>/dev/null
	docker run \
      --rm \
      -t \
      -d \
      --name $(WORKER_NAME) \
      -h $(WORKER_NAME) \
      -p $(SSH_HOST_PORT):22 \
      -v "$(CRAN_DIR):$(CRAN_DIR):ro" \
      -v "$(LIBRARY_DIR):$(LIBRARY_DIR)" \
      -v "$(CURDIR):$(CURDIR)" \
      -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent \
      -e SSH_AUTH_SOCK=/ssh-agent \
      -e R_LIBS="$(CURDIR)/library:$(LIBRARY_DIR)" \
      -e NO_EXEC_SUDO=1 \
      -e USER_ID=$$(id -u) \
      -e USER_GID=$$(id -g) \
      $(DOCKER_IMAGE_NAME) \
      /usr/sbin/sshd -De

.node-status:
	-@ppid=$$(pgrep parallel); \
  tasks=$$([ $$? -eq 0 ] && ps -o pid= --ppid $$ppid | wc -l || echo 0); \
  load=$$(uptime | sed "s/.*load average: //"); \
  echo "$(HOSTNAME): load: $$load, tasks: $$tasks"
	@TERM=xterm make envir

.node-update:
	make -C $(CURDIR) libs

NODES_ALL := $(patsubst %, node/%, $(HOSTS))
HOSTS_ALL := $(patsubst %, host/%, $(HOSTS))
HOSTS_SLAVES := $(patsubst %, host/%, $(SLAVES))

# this seemingly empty target is important, because
# it allows one to use $< in the remore/% target
$(HOSTS):

ssh/%: %
	$(SSH) $<

host/%: %
	@ssh $< $(TARGET)

node/%: %
	@$(SSH) $< $(TARGET)

.PHONY: hosts-all
hosts-all: $(HOSTS_ALL)

.PHONY: hosts-slaves
hosts-slaves: $(HOSTS_SLAVES)

.PHONY: nodes-all
nodes-all: $(NODES_ALL)

.PHONY: r-dyntrace-docker-image
r-dyntrace-docker-image: TARGET=$(THIS_MAKE) .r-dyntrace-docker-image
r-dyntrace-docker-image: hosts-slaves

.PHONY: node-setup
node-setup: TARGET=$(THIS_MAKE) .node-setup
node-setup: hosts-all
	if tmux list-windows -F "#W" | grep cluster >/dev/null 2>&1; then tmux kill-window -t cluster; else true; fi
	tmux new-window -n cluster -d
	sleep 1
	$(foreach host,$(HOSTS),tmux split-window -d -t cluster $(THIS_MAKE) node/$(host) SSH_EXTRA="-MT"; sleep 1;)
	tmux kill-pane -t cluster.1
	tmux select-layout -t cluster even-vertical

.PHONY: node-status
node-status: TARGET=$(THIS_MAKE) .node-status
node-status: nodes-all

.PHONY: node-update
node-update: TARGET=$(THIS_MAKE) .node-update
node-update: nodes-all

$(SSH_LOGIN_FILE):
	-@rm -f $(SSH_LOGIN_FILE)
	@$(foreach host,$(HOSTS),echo "$(SSH_NUM_SLOTS)/$(SSH) $(host)" >> $(SSH_LOGIN_FILE);)