Merge pull request #91 from populationgenomics/add-version-endpoint-v3

Add version endpoint (third time's a charm)
daniel-goldstein · Mar 15, 2021 · c78cf7e · c78cf7e
2 parents d65e318 + bff3779
commit c78cf7e
Show file tree

Hide file tree

Showing 14 changed files with 81 additions and 51 deletions.
diff --git a/batch/Dockerfile.worker b/batch/Dockerfile.worker
@@ -34,6 +34,7 @@ RUN python3 -m pip install --upgrade --no-cache-dir pip
 
 COPY hail/python/setup-hailtop.py /hailtop/setup.py
 COPY hail/python/hailtop /hailtop/hailtop/
+COPY /hail_version /hailtop/hailtop/hail_version
 RUN hail-pip-install --no-deps /hailtop && rm -rf /hailtop
 
 COPY gear/setup.py /gear/setup.py

diff --git a/build.yaml b/build.yaml
@@ -57,13 +57,6 @@ steps:
    publishAs: base
    dependsOn:
     - hail_ubuntu_image
- - kind: buildImage
-   name: service_base_image
-   dockerFile: docker/Dockerfile.service-base
-   contextPath: .
-   publishAs: service-base
-   dependsOn:
-    - base_image
  - kind: runImage
    name: check_hail
    image:
@@ -76,21 +69,6 @@ steps:
      make check-hail
    dependsOn:
      - base_image
- - kind: runImage
-   name: check_services
-   image:
-     valueFrom: service_base_image.image
-   script: |
-     set -ex
-     mkdir repo
-     cd repo
-     {{ code.checkout_script }}
-     {% if 'target_sha' in code %}
-     export HAIL_TARGET_SHA={{ code.target_sha }}
-     {% endif %}
-     make -k check-services
-   dependsOn:
-     - service_base_image
  - kind: runImage
    name: copy_files
    image:
@@ -142,6 +120,32 @@ steps:
        to: /repo/ci/create_initial_account.py
    dependsOn:
      - base_image
+ - kind: buildImage
+   name: service_base_image
+   dockerFile: docker/Dockerfile.service-base
+   contextPath: .
+   publishAs: service-base
+   dependsOn:
+    - base_image
+    - copy_files
+   inputs:
+    - from: /hail_version
+      to: /hail_version
+ - kind: runImage
+   name: check_services
+   image:
+     valueFrom: service_base_image.image
+   script: |
+     set -ex
+     mkdir repo
+     cd repo
+     {{ code.checkout_script }}
+     {% if 'target_sha' in code %}
+     export HAIL_TARGET_SHA={{ code.target_sha }}
+     {% endif %}
+     make -k check-services
+   dependsOn:
+     - service_base_image
  - kind: buildImage
    name: create_certs_image
    dockerFile: tls/Dockerfile
@@ -555,8 +559,12 @@ steps:
    scopes:
     - test
     - dev
+   inputs:
+    - from: /hail_version
+      to: /hail_version
    dependsOn:
      - base_image
+     - copy_files
  - kind: buildImage
    name: test_monitoring_image
    dockerFile: monitoring/Dockerfile.test
@@ -704,15 +712,22 @@ steps:
    inputs:
      - from: /just-jar/hail.jar
        to: /batch/hail.jar
+     - from: /hail_version
+       to: /hail_version
    dependsOn:
      - build_hail_jar_only
+     - copy_files
  - kind: buildImage
    name: service_java_run_base_image
    dockerFile: docker/Dockerfile.service-java-run-base
    contextPath: .
    publishAs: service-java-run-base
    dependsOn:
      - hail_ubuntu_image
+     - copy_files
+   inputs:
+     - from: /hail_version
+       to: /hail_version
  - kind: buildImage
    name: query_image
    dockerFile: query/Dockerfile

diff --git a/docker/Dockerfile.service-base b/docker/Dockerfile.service-base
@@ -7,11 +7,8 @@ RUN hail-pip-install -r service-base-requirements.txt
 
 COPY hail/python/setup-hailtop.py /hailtop/setup.py
 COPY hail/python/hailtop /hailtop/hailtop/
-COPY hail/Makefile hail/env_var.mk .git/ hailtop/
-RUN mkdir -p hailtop/python/hail hailtop/python/hailtop/hailctl hailtop/python/hail/docs/_static && \
-    (cd hailtop && echo $(pwd) && make python-version-info) && \
-    cp hailtop/python/hail/hail_*version hailtop/hailtop/hailctl && \
-    rm -rf hailtop/Makefile hailtop/env_var.mk .git/
+COPY /hail_version /hailtop/hailtop/hail_version
+COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in
 RUN hail-pip-install /hailtop && rm -rf /hailtop
 
 COPY gear/setup.py /gear/setup.py
@@ -21,3 +18,4 @@ RUN hail-pip-install /gear && rm -rf /gear
 COPY web_common/setup.py web_common/MANIFEST.in /web_common/
 COPY web_common/web_common /web_common/web_common/
 RUN hail-pip-install /web_common && rm -rf /web_common
+RUN echo 'from hailtop import version; print(version());' | python3
diff --git a/docker/Dockerfile.service-java-run-base b/docker/Dockerfile.service-java-run-base
@@ -24,7 +24,10 @@ RUN hail-pip-install -r service-base-requirements.txt
 
 COPY hail/python/setup-hailtop.py /hailtop/setup.py
 COPY hail/python/hailtop /hailtop/hailtop/
+COPY /hail_version /hailtop/hailtop/hail_version
+COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in
 RUN hail-pip-install /hailtop && rm -rf /hailtop
+RUN echo 'from hailtop import version; print(version());' | python3
 
 COPY gear/setup.py /gear/setup.py
 COPY gear/gear /gear/gear/

diff --git a/docker/Makefile b/docker/Makefile
@@ -48,6 +48,8 @@ base-stmp: hail-ubuntu-stmp Dockerfile.base core-site.xml requirements.txt ../py
 .PHONY: service-base
 service-base: base-stmp
 	-docker pull $(SERVICE_BASE_LATEST)
+	$(MAKE) -C ../hail python/hailtop/hail_version
+	cp ../hail/python/hailtop/hail_version /hail_version
 	python3 ../ci/jinja2_render.py '{"base_image":{"image":"base"}}' Dockerfile.service-base Dockerfile.service-base.out
 	[ "$(shell bash stat-permissions.sh Dockerfile.service-base.out)" = "644" ]
 	[ "$(shell bash stat-permissions.sh service-base-requirements.txt)" = "644" ]
@@ -56,6 +58,8 @@ service-base: base-stmp
 .PHONY: service-java-run-base
 service-java-run-base: hail-ubuntu-stmp
 	-docker pull $(SERVICE_JAVA_RUN_BASE_LATEST)
+	$(MAKE) -C ../hail python/hailtop/hail_version
+	cp ../hail/python/hailtop/hail_version /hail_version
 	python3 ../ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"hail-ubuntu"}}' Dockerfile.service-java-run-base Dockerfile.service-java-run-base.out
 	[ "$(shell bash stat-permissions.sh Dockerfile.service-java-run-base.out)" = "644" ]
 	docker build -t service-java-run-base -f Dockerfile.service-java-run-base.out --cache-from service-java-run-base,$(SERVICE_JAVA_RUN_BASE_LATEST),base,hail-ubuntu ..

diff --git a/hail/Dockerfile.hailtop b/hail/Dockerfile.hailtop
diff --git a/hail/Makefile b/hail/Makefile
@@ -50,7 +50,7 @@ PY_FILES := $(shell git ls-files python)
 INIT_SCRIPTS := python/hailtop/hailctl/deploy.yaml
 PYTHON_VERSION_INFO := python/hail/hail_version
 PYTHON_VERSION_INFO += python/hail/hail_pip_version
-PYTHON_VERSION_INFO += python/hailtop/hailctl/hail_version
+PYTHON_VERSION_INFO += python/hailtop/hail_version
 PYTHON_VERSION_INFO += python/hail/docs/_static/hail_version.js
 SCALA_BUILD_INFO := src/main/resources/build-info.properties
 
@@ -128,7 +128,7 @@ python/hail/docs/_static/hail_version.js: python/hail/hail_version python/hail/h
 	printf 'hail_version="$(shell cat python/hail/hail_version)";' > $@
 	printf 'hail_pip_version="$(shell cat python/hail/hail_pip_version)"' >> $@
 
-python/hailtop/hailctl/hail_version: python/hail/hail_version
+python/hailtop/hail_version: python/hail/hail_version
 	cp -f $< $@
 
 python/README.md: ../README.md

diff --git a/hail/python/MANIFEST.in b/hail/python/MANIFEST.in
@@ -0,0 +1,6 @@
+include hail/hail_pip_version
+include hail/hail_version
+include hail/experimental/datasets.json
+include hail/backend/hail-all-spark.jar
+include hailtop/hail_version
+include hailtop/hailctl/deploy.yaml
diff --git a/hail/python/hailtop/__init__.py b/hail/python/hailtop/__init__.py
@@ -0,0 +1,8 @@
+import pkg_resources
+
+
+_VERSION = pkg_resources.resource_string(__name__, 'hail_version').decode().strip()
+
+
+def version() -> str:
+    return _VERSION
diff --git a/hail/python/hailtop/hailctl/__init__.py b/hail/python/hailtop/hailctl/__init__.py
@@ -1,8 +0,0 @@
-def version() -> str:
-    import pkg_resources  # pylint: disable=import-outside-toplevel
-    return pkg_resources.resource_string(__name__, 'hail_version').decode().strip()
-
-
-__all__ = [
-    'version'
-]

diff --git a/hail/python/hailtop/hailctl/__main__.py b/hail/python/hailtop/hailctl/__main__.py
@@ -6,6 +6,7 @@
 import time
 
 from hailtop import hailctl
+from hailtop import version
 
 
 def print_help():
@@ -61,7 +62,7 @@ def check_for_update():
                 pip_out = sp.check_output(['pip3', 'search', 'hail'], stderr=sp.STDOUT)
 
             latest = re.search(r'hail \((\\d+)\.(\\d+)\.(\\d+).*', pip_out.decode()).groups()
-            installed = re.search(r'(\d+)\.(\d+)\.(\d+).*', hailctl.version()).groups()
+            installed = re.search(r'(\d+)\.(\d+)\.(\d+).*', version()).groups()
 
             def int_version(version):
                 return tuple(map(int, version))
@@ -81,7 +82,7 @@ def fmt_version(version):
 
 
 def print_version():
-    print(hailctl.version())
+    print(version())
 
 
 def main():

diff --git a/hail/python/setup-hailtop.py b/hail/python/setup-hailtop.py
@@ -17,8 +17,9 @@
     package_dir={
         'hailtop': 'hailtop'},
     package_data={
-        "hailtop": ["py.typed"],
-        'hailtop.hailctl': ['hail_version', 'deploy.yaml']},
+        "hailtop": ["py.typed", "hail_version"],
+        'hailtop.hailctl': ['hail_version', 'deploy.yaml']
+    },
     classifiers=[
         "Programming Language :: Python :: 3",
         "License :: OSI Approved :: MIT License",
@@ -27,5 +28,6 @@
     entry_points={
         'console_scripts': ['hailctl = hailtop.hailctl.__main__:main']
     },
-    setup_requires=["pytest-runner", "wheel"]
+    setup_requires=["pytest-runner", "wheel"],
+    include_package_data=True,
 )
diff --git a/hail/python/setup.py b/hail/python/setup.py
@@ -41,6 +41,7 @@
                  'hail_version',
                  'experimental/datasets.json'],
         'hail.backend': ['hail-all-spark.jar'],
+        'hailtop': ['hail_version'],
         'hailtop.hailctl': ['hail_version', 'deploy.yaml']},
     classifiers=[
         "Programming Language :: Python :: 3",
@@ -52,5 +53,6 @@
         'console_scripts': ['hailctl = hailtop.hailctl.__main__:main']
     },
     setup_requires=["pytest-runner", "wheel"],
-    tests_require=["pytest"]
+    tests_require=["pytest"],
+    include_package_data=True,
 )
diff --git a/query/query/query.py b/query/query/query.py
@@ -13,7 +13,7 @@
 from hailtop.config import get_deploy_config
 from hailtop.tls import internal_server_ssl_context
 from hailtop.hail_logging import AccessLogger
-from hailtop.hailctl import version
+from hailtop import version
 from gear import (
     setup_aiohttp_session,
     rest_authenticated_users_only,
@@ -22,8 +22,6 @@
 
 from .sockets import connect_to_java
 
-from .sockets import connect_to_java
-
 uvloop.install()
 
 DEFAULT_NAMESPACE = os.environ['HAIL_DEFAULT_NAMESPACE']
@@ -241,6 +239,11 @@ async def set_flag(request, userdata):  # pylint: disable=unused-argument
     return web.json_response(jresp)
 
 
+@routes.get('/api/v1alpha/version')
+async def rest_get_version(request):  # pylint: disable=W0613
+    return web.Response(text=version())
+
+
 async def on_startup(app):
     thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=16)
     app['thread_pool'] = thread_pool