Merge pull request #46 from iterait/dev

Release 0.4.0

.circleci/config.yml

@@ -9,22 +9,23 @@ references:
         export DEBIAN_FRONTEND=noninteractive
         apt-get update -qy
         apt-get install -y git python3-dev python3-pip curl
+        pip3 install -r requirements.txt
         pip3 install -e .
-        echo "$(tail -n +2 cxworker/manage.py)" > cxworker/manage.py  # remove gevent monkey patch (it breaks sphinx)
+        echo "$(tail -n +2 shepherd/manage.py)" > shepherd/manage.py  # remove gevent monkey patch (it breaks sphinx)
 
   install: &install
     run:
-      name: Install dependencies and cxworker on Archlinux
+      name: Install dependencies and shepherd on Archlinux
       command: |
         set -x
         VER="17.11.0-ce"
         curl -L -o /tmp/docker-$VER.tgz https://download.docker.com/linux/static/test/x86_64/docker-$VER.tgz
         tar -xz -C /tmp -f /tmp/docker-$VER.tgz
         mv /tmp/docker/* /usr/bin
         pacman --noconfirm --needed -Syu minio
-        mkdir examples/docker/cxflow_example/ssh
-        echo $PRIVATE_SSH_KEY_PRIVATE | tr ' ' '\n' | base64 -d > examples/docker/cxflow_example/ssh/id_rsa
-        pip install -e .
+        mkdir examples/docker/emloop_example/ssh
+        pip3 install -r requirements.txt
+        pip3 install -e .
 
   add_rw_key: &add_rw_key
     add_ssh_keys:
@@ -35,20 +36,23 @@ references:
     run:
       name: Run tests.
       command: |
-        python setup.py test
+        useradd -m -s /bin/bash test
+        passwd -d test
+        chown -R test /root
+        sudo -u test pytest --forked
 
   doc: &doc
     run:
       name: Generate documentation
       command: |
-        bash <(curl -fsSL https://raw.githubusercontent.com/Cognexa/ci-utils/master/doc_deploy.sh)
+        bash <(curl -fsSL https://raw.githubusercontent.com/iterait/ci-utils/master/doc_deploy.sh)
 
 jobs:
 
   test_archlinux:
     docker:
-      - image: cognexa/cxflow:latest
-    working_directory: ~/cxworker
+      - image: iterait/emloop
+    working_directory: ~/shepherd
     steps:
       - setup_remote_docker:
           docker_layer_caching: true
@@ -60,7 +64,7 @@ jobs:
   generate_documentation:
     docker:
       - image: ubuntu:rolling
-    working_directory: ~/cxworker
+    working_directory: ~/shepherd
     steps:
       - checkout
       - *ubuntu_install
@@ -72,9 +76,21 @@ workflows:
   test:
     jobs:
       - test_archlinux
-      - generate_documentation:
+#      - generate_documentation:
+#          filters:
+#            branches:
+#              only: dev
+#          requires:
+#            - test_archlinux
+
+  nightly-build:
+    triggers:
+      - schedule:
+          cron: "0 0 * * *"
           filters:
             branches:
-              only: dev
-          requires:
-            - test_archlinux
+              only:
+                - master
+                - dev
+    jobs:
+      - test_archlinux

.gitignore

@@ -103,7 +103,7 @@ ENV/
 .mypy_cache/
 
 ssh
-cxworker.*.rst
-docs/cxworker/cxworker.rst
+shepherd.*.rst
+docs/shepherd/shepherd.rst
 .pytest_cache
 

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "docs/_base"]
 	path = docs/_base
-	url = https://github.com/Cognexa/sphinx-docs
+	url = https://github.com/iterait/sphinx-docs

LICENSE

@@ -1,6 +1,7 @@
 MIT License
 
-Copyright (c) 2018 Cognexa Solutions s.r.o.
+Copyright (c) 9/2018 - present  Iterait a.s.
+              4/2018 - 8/2018   Cognexa Solutions s.r.o.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,4 +1,6 @@
-# CXWorker
+# Shepherd
+
+[![CircleCI](https://circleci.com/gh/iterait/shepherd.png?style=shield&circle-token=1045f8994f4f35d81130331600a0683e16bbb4f9)](https://circleci.com/gh/iterait/shepherd/tree/master)
 
 Provides access to computation resources on a single machine.
 
@@ -15,7 +17,7 @@ Provides access to computation resources on a single machine.
 
 The test suite can be run with `python setup.py test`.
 
-### Launching the Worker
+### Launching the Shepherd
 
 First, you need to have a Docker registry and a Minio server running. The 
 easiest way to achieve this is to use the Docker Compose example:
@@ -28,18 +30,18 @@ Second, you need a configuration file. Again, examples found in the `examples/co
 folder are a great starting point. Feel free to pick one of those and edit it to 
 your needs.
 
-Finally, you need to run the following command to start the worker:
+Finally, you need to run the following command to start the shepherd:
 
 ```
-cxworker -c examples/configs/cxworker-docker-cpu.yml
+shepherd -c examples/configs/shepherd-docker-cpu.yml
 ```
 
 Be sure to adjust the command line parameters according to your needs (`-h` is 
-your host address, `-p` is the port number where the worker API server listens 
+your host address, `-p` is the port number where the shepherd API server listens 
 and `-c` is the path to the configuration file).
 
-After launching the worker, there will be an HTTP API available on the 
-configured port that can be used to control the worker.
+After launching the shepherd, there will be an HTTP API available on the 
+configured port that can be used to control the shepherd.
 
 ### Processing a Request Directly
 

docs/Makefile

@@ -4,7 +4,7 @@
 # You can set these variables from the command line.
 SPHINXOPTS    =
 SPHINXBUILD   = python -msphinx
-SPHINXPROJ    = cxflow
+SPHINXPROJ    = emloop
 SOURCEDIR     = .
 BUILDDIR      = _build
 

docs/api.rst

@@ -1,4 +1,4 @@
-cxworker API
+Shepherd API
 ============
 
 TBD

docs/bare_sheep.rst

@@ -1,8 +1,8 @@
 Bare Sheep
 ==========
 
-:py:class:`cxworker.sheep.BareSheep` uses a sub-process to run its *runner*.
-In particular, it runs ``cxworker-runner`` command which is linked to :py:func:`cxworker.runner.runner_entry_point.run`.
+:py:class:`shepherd.sheep.BareSheep` uses a sub-process to run its *runner*.
+In particular, it runs ``shepherd-runner`` command which is linked to :py:func:`shepherd.runner.runner_entry_point.run`.
 
 As mentioned earlier, a sheep communicates with its runner via a socket.
 The messages are minimal though, in principle, only ``job_id`` s are passed through the socket while the data are
@@ -18,19 +18,19 @@ Bare sheep's config is fairly simple:
   bare_sheep:
     port: 9001
     type: bare
-    working_directory: examples/docker/cxflow_example
-    stdout_file: /tmp/bare-cxworker-runner-stdout.txt
-    stderr_file: /tmp/bare-cxworker-runner-stderr.txt
+    working_directory: examples/docker/emloop_example
+    stdout_file: /tmp/bare-shepherd-runner-stdout.txt
+    stderr_file: /tmp/bare-shepherd-runner-stderr.txt
 
 Aside from configuration common for all sheep (socket ``port`` and sheep ``type``), bare sheep allows to configure:
 
-- ``working_directory`` directory from which ``cxworker-runner`` command is called
+- ``working_directory`` directory from which ``shepherd-runner`` command is called
 - ``stdout_file`` and ``stderr_file`` to store the **runner** outputs
 
 Model Name and Version
 **********************
 
-Bare sheep allows to run *jobs* with any model name and version as long as a **cxflow** configuration file can be
+Bare sheep allows to run *jobs* with any model name and version as long as a **emloop** configuration file can be
 found in ``working_directory``/``model_name``/``model_version``/``config.yaml``.
 
 For example with ``working_directory="/var"``, ``model_name="my_project/models"`` and finally ``model_version`` empty,

docs/conf.py

@@ -5,28 +5,28 @@
 from conf import *
 
 autoapi_modules = {
-    'cxworker': {
+    'shepherd': {
         # 'override': True,
         # 'output': 'auto',
         'prune': True
     }
 }
 
 # General information about the project.
-project = 'cxworker'
-copyright = '2017, Cognexa Solutions s.r.o.'
+project = 'shepherd'
+copyright = '2018, Iterait a.s.'
 author = 'Jan Buchar, Adam Blazek, Petr Belohlavek'
 
 # The short X.Y version.
-version = '.'.join(pkg_resources.get_distribution("cxworker").version.split('.')[:2])
+version = '.'.join(pkg_resources.get_distribution("shepherd").version.split('.')[:2])
 # The full version, including alpha/beta/rc tags.
-release = pkg_resources.get_distribution("cxworker").version
+release = pkg_resources.get_distribution("shepherd").version
 
 html_context.update(analytics_id="UA-108491604-2")
 
 html_theme_options.update({
     # Navigation bar title. (Default: ``project`` value)
-    'navbar_title': "cxworker",
+    'navbar_title': "shepherd",
 
     # Tab name for entire site. (Default: "Site")
     'navbar_site_name': "Pages",
@@ -38,7 +38,7 @@
         ("Docker Sheep", "docker_sheep"),
         ("Runners", "runners"),
         ("API", "api"),
-        ("Package Reference", "cxworker/index"),
+        ("Package Reference", "shepherd/index"),
     ],
 
     # HTML navbar class (Default: "navbar") to attach to <div> element.

docs/docker_sheep.rst

@@ -1,7 +1,7 @@
 Docker Sheep
 ============
 
-:py:class:`cxworker.sheep.DockerSheep` does exactly the same as `bare sheep <bare_sheep.html>`_ except it
+:py:class:`shepherd.sheep.DockerSheep` does exactly the same as `bare sheep <bare_sheep.html>`_ except it
 runs the *runner* in a docker. Similarly to the bare sheep's runner, docker runner is expected to listen for
 incoming jobs at a socket bind to port **9999**.
 
@@ -27,5 +27,5 @@ When new model name and version is encountered, docker sheep pulls the docker im
 
 Example Dockerfile follows:
 
-.. include:: ../examples/docker/cxflow_example/Dockerfile
+.. include:: ../examples/docker/emloop_example/Dockerfile
    :literal: