From 7f38eb653a00775594cd0e99b49a667bc6956e4c Mon Sep 17 00:00:00 2001 From: Michael Franklin Date: Mon, 15 Mar 2021 10:57:37 +1100 Subject: [PATCH 1/5] Revert "Revert "[query] Add version endpoint (again)"" This reverts commit 15669bdc85f030f58d422fc9b0ca77282ca5fd5b. --- batch/Dockerfile.worker | 1 + build.yaml | 55 +++++++++++++++---------- docker/Dockerfile.service-base | 8 ++-- docker/Makefile | 2 + hail/Dockerfile.hailtop | 5 --- hail/Makefile | 4 +- hail/python/MANIFEST.in | 6 +++ hail/python/hailtop/__init__.py | 8 ++++ hail/python/hailtop/hailctl/__init__.py | 8 ---- hail/python/hailtop/hailctl/__main__.py | 5 ++- hail/python/setup-hailtop.py | 8 ++-- hail/python/setup.py | 4 +- query/query/query.py | 9 ++-- 13 files changed, 72 insertions(+), 51 deletions(-) delete mode 100644 hail/Dockerfile.hailtop create mode 100644 hail/python/MANIFEST.in diff --git a/batch/Dockerfile.worker b/batch/Dockerfile.worker index 9af5ad90937..2ef29ed6aed 100644 --- a/batch/Dockerfile.worker +++ b/batch/Dockerfile.worker @@ -34,6 +34,7 @@ RUN python3 -m pip install --upgrade --no-cache-dir pip COPY hail/python/setup-hailtop.py /hailtop/setup.py COPY hail/python/hailtop /hailtop/hailtop/ +COPY /hail_version /hailtop/hailtop/hail_version RUN hail-pip-install --no-deps /hailtop && rm -rf /hailtop COPY gear/setup.py /gear/setup.py diff --git a/build.yaml b/build.yaml index ada34669e7c..dd78eb4b097 100644 --- a/build.yaml +++ b/build.yaml @@ -57,13 +57,6 @@ steps: publishAs: base dependsOn: - hail_ubuntu_image - - kind: buildImage - name: service_base_image - dockerFile: docker/Dockerfile.service-base - contextPath: . - publishAs: service-base - dependsOn: - - base_image - kind: runImage name: check_hail image: @@ -76,21 +69,6 @@ steps: make check-hail dependsOn: - base_image - - kind: runImage - name: check_services - image: - valueFrom: service_base_image.image - script: | - set -ex - mkdir repo - cd repo - {{ code.checkout_script }} - {% if 'target_sha' in code %} - export HAIL_TARGET_SHA={{ code.target_sha }} - {% endif %} - make -k check-services - dependsOn: - - service_base_image - kind: runImage name: copy_files image: @@ -142,6 +120,32 @@ steps: to: /repo/ci/create_initial_account.py dependsOn: - base_image + - kind: buildImage + name: service_base_image + dockerFile: docker/Dockerfile.service-base + contextPath: . + publishAs: service-base + dependsOn: + - base_image + - copy_files + inputs: + - from: /hail_version + to: /hail_version + - kind: runImage + name: check_services + image: + valueFrom: service_base_image.image + script: | + set -ex + mkdir repo + cd repo + {{ code.checkout_script }} + {% if 'target_sha' in code %} + export HAIL_TARGET_SHA={{ code.target_sha }} + {% endif %} + make -k check-services + dependsOn: + - service_base_image - kind: buildImage name: create_certs_image dockerFile: tls/Dockerfile @@ -555,8 +559,12 @@ steps: scopes: - test - dev + inputs: + - from: /hail_version + to: /hail_version dependsOn: - base_image + - copy_files - kind: buildImage name: test_monitoring_image dockerFile: monitoring/Dockerfile.test @@ -704,8 +712,11 @@ steps: inputs: - from: /just-jar/hail.jar to: /batch/hail.jar + - from: /hail_version + to: /hail_version dependsOn: - build_hail_jar_only + - copy_files - kind: buildImage name: service_java_run_base_image dockerFile: docker/Dockerfile.service-java-run-base diff --git a/docker/Dockerfile.service-base b/docker/Dockerfile.service-base index dfd3e7bb779..cabb9ae0864 100644 --- a/docker/Dockerfile.service-base +++ b/docker/Dockerfile.service-base @@ -7,12 +7,10 @@ RUN hail-pip-install -r service-base-requirements.txt COPY hail/python/setup-hailtop.py /hailtop/setup.py COPY hail/python/hailtop /hailtop/hailtop/ -COPY hail/Makefile hail/env_var.mk .git/ hailtop/ -RUN mkdir -p hailtop/python/hail hailtop/python/hailtop/hailctl hailtop/python/hail/docs/_static && \ - (cd hailtop && echo $(pwd) && make python-version-info) && \ - cp hailtop/python/hail/hail_*version hailtop/hailtop/hailctl && \ - rm -rf hailtop/Makefile hailtop/env_var.mk .git/ +COPY /hail_version /hailtop/hailtop/hail_version +COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in RUN hail-pip-install /hailtop && rm -rf /hailtop +RUN echo 'from hailtop import version; print(version());' | python3 COPY gear/setup.py /gear/setup.py COPY gear/gear /gear/gear/ diff --git a/docker/Makefile b/docker/Makefile index e796702023c..0111fbe8d9b 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -48,6 +48,8 @@ base-stmp: hail-ubuntu-stmp Dockerfile.base core-site.xml requirements.txt ../py .PHONY: service-base service-base: base-stmp -docker pull $(SERVICE_BASE_LATEST) + $(MAKE) -C ../hail python/hailtop/hail_version + cp ../hail/python/hailtop/hail_version /hail_version python3 ../ci/jinja2_render.py '{"base_image":{"image":"base"}}' Dockerfile.service-base Dockerfile.service-base.out [ "$(shell bash stat-permissions.sh Dockerfile.service-base.out)" = "644" ] [ "$(shell bash stat-permissions.sh service-base-requirements.txt)" = "644" ] diff --git a/hail/Dockerfile.hailtop b/hail/Dockerfile.hailtop deleted file mode 100644 index 6e39fbe3a3d..00000000000 --- a/hail/Dockerfile.hailtop +++ /dev/null @@ -1,5 +0,0 @@ -FROM {{ base_image.image }} - -COPY python/setup-hailtop.py /hailtop/setup.py -COPY python/hailtop /hailtop/hailtop/ -RUN hail-pip-install /hailtop && rm -rf /hailtop diff --git a/hail/Makefile b/hail/Makefile index b0570f3eb93..36c57a99717 100644 --- a/hail/Makefile +++ b/hail/Makefile @@ -50,7 +50,7 @@ PY_FILES := $(shell git ls-files python) INIT_SCRIPTS := python/hailtop/hailctl/deploy.yaml PYTHON_VERSION_INFO := python/hail/hail_version PYTHON_VERSION_INFO += python/hail/hail_pip_version -PYTHON_VERSION_INFO += python/hailtop/hailctl/hail_version +PYTHON_VERSION_INFO += python/hailtop/hail_version PYTHON_VERSION_INFO += python/hail/docs/_static/hail_version.js SCALA_BUILD_INFO := src/main/resources/build-info.properties @@ -128,7 +128,7 @@ python/hail/docs/_static/hail_version.js: python/hail/hail_version python/hail/h printf 'hail_version="$(shell cat python/hail/hail_version)";' > $@ printf 'hail_pip_version="$(shell cat python/hail/hail_pip_version)"' >> $@ -python/hailtop/hailctl/hail_version: python/hail/hail_version +python/hailtop/hail_version: python/hail/hail_version cp -f $< $@ python/README.md: ../README.md diff --git a/hail/python/MANIFEST.in b/hail/python/MANIFEST.in new file mode 100644 index 00000000000..9016d2389cb --- /dev/null +++ b/hail/python/MANIFEST.in @@ -0,0 +1,6 @@ +include hail/hail_pip_version +include hail/hail_version +include hail/experimental/datasets.json +include hail/backend/hail-all-spark.jar +include hailtop/hail_version +include hailtop/hailctl/deploy.yaml diff --git a/hail/python/hailtop/__init__.py b/hail/python/hailtop/__init__.py index e69de29bb2d..603455b7801 100644 --- a/hail/python/hailtop/__init__.py +++ b/hail/python/hailtop/__init__.py @@ -0,0 +1,8 @@ +import pkg_resources + + +_VERSION = pkg_resources.resource_string(__name__, 'hail_version').decode().strip() + + +def version() -> str: + return _VERSION diff --git a/hail/python/hailtop/hailctl/__init__.py b/hail/python/hailtop/hailctl/__init__.py index 96598044c3e..e69de29bb2d 100644 --- a/hail/python/hailtop/hailctl/__init__.py +++ b/hail/python/hailtop/hailctl/__init__.py @@ -1,8 +0,0 @@ -def version() -> str: - import pkg_resources # pylint: disable=import-outside-toplevel - return pkg_resources.resource_string(__name__, 'hail_version').decode().strip() - - -__all__ = [ - 'version' -] diff --git a/hail/python/hailtop/hailctl/__main__.py b/hail/python/hailtop/hailctl/__main__.py index 66ecfff66b1..ed6b61a66c8 100644 --- a/hail/python/hailtop/hailctl/__main__.py +++ b/hail/python/hailtop/hailctl/__main__.py @@ -6,6 +6,7 @@ import time from hailtop import hailctl +from hailtop import version def print_help(): @@ -61,7 +62,7 @@ def check_for_update(): pip_out = sp.check_output(['pip3', 'search', 'hail'], stderr=sp.STDOUT) latest = re.search(r'hail \((\\d+)\.(\\d+)\.(\\d+).*', pip_out.decode()).groups() - installed = re.search(r'(\d+)\.(\d+)\.(\d+).*', hailctl.version()).groups() + installed = re.search(r'(\d+)\.(\d+)\.(\d+).*', version()).groups() def int_version(version): return tuple(map(int, version)) @@ -81,7 +82,7 @@ def fmt_version(version): def print_version(): - print(hailctl.version()) + print(version()) def main(): diff --git a/hail/python/setup-hailtop.py b/hail/python/setup-hailtop.py index 780e6da3e89..1804909364a 100644 --- a/hail/python/setup-hailtop.py +++ b/hail/python/setup-hailtop.py @@ -17,8 +17,9 @@ package_dir={ 'hailtop': 'hailtop'}, package_data={ - "hailtop": ["py.typed"], - 'hailtop.hailctl': ['hail_version', 'deploy.yaml']}, + "hailtop": ["py.typed", "hail_version"], + 'hailtop.hailctl': ['hail_version', 'deploy.yaml'] + }, classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", @@ -27,5 +28,6 @@ entry_points={ 'console_scripts': ['hailctl = hailtop.hailctl.__main__:main'] }, - setup_requires=["pytest-runner", "wheel"] + setup_requires=["pytest-runner", "wheel"], + include_package_data=True, ) diff --git a/hail/python/setup.py b/hail/python/setup.py index bcaf3c7337c..84d7a8b791a 100755 --- a/hail/python/setup.py +++ b/hail/python/setup.py @@ -41,6 +41,7 @@ 'hail_version', 'experimental/datasets.json'], 'hail.backend': ['hail-all-spark.jar'], + 'hailtop': ['hail_version'], 'hailtop.hailctl': ['hail_version', 'deploy.yaml']}, classifiers=[ "Programming Language :: Python :: 3", @@ -52,5 +53,6 @@ 'console_scripts': ['hailctl = hailtop.hailctl.__main__:main'] }, setup_requires=["pytest-runner", "wheel"], - tests_require=["pytest"] + tests_require=["pytest"], + include_package_data=True, ) diff --git a/query/query/query.py b/query/query/query.py index 0a394accdcd..a6ed04646b9 100644 --- a/query/query/query.py +++ b/query/query/query.py @@ -13,7 +13,7 @@ from hailtop.config import get_deploy_config from hailtop.tls import internal_server_ssl_context from hailtop.hail_logging import AccessLogger -from hailtop.hailctl import version +from hailtop import version from gear import ( setup_aiohttp_session, rest_authenticated_users_only, @@ -22,8 +22,6 @@ from .sockets import connect_to_java -from .sockets import connect_to_java - uvloop.install() DEFAULT_NAMESPACE = os.environ['HAIL_DEFAULT_NAMESPACE'] @@ -241,6 +239,11 @@ async def set_flag(request, userdata): # pylint: disable=unused-argument return web.json_response(jresp) +@routes.get('/api/v1alpha/version') +async def rest_get_version(request): # pylint: disable=W0613 + return web.Response(text=version()) + + async def on_startup(app): thread_pool = concurrent.futures.ThreadPoolExecutor(max_workers=16) app['thread_pool'] = thread_pool From 50e0836d8f47fe79aeaf7ee12286c1049c6d664e Mon Sep 17 00:00:00 2001 From: MIchael Franklin Date: Mon, 15 Mar 2021 09:20:14 +1100 Subject: [PATCH 2/5] Ensure hailtop.version is present at end of install --- docker/Dockerfile.service-base | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile.service-base b/docker/Dockerfile.service-base index cabb9ae0864..c4c9fbfb6ca 100644 --- a/docker/Dockerfile.service-base +++ b/docker/Dockerfile.service-base @@ -10,7 +10,6 @@ COPY hail/python/hailtop /hailtop/hailtop/ COPY /hail_version /hailtop/hailtop/hail_version COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in RUN hail-pip-install /hailtop && rm -rf /hailtop -RUN echo 'from hailtop import version; print(version());' | python3 COPY gear/setup.py /gear/setup.py COPY gear/gear /gear/gear/ @@ -19,3 +18,4 @@ RUN hail-pip-install /gear && rm -rf /gear COPY web_common/setup.py web_common/MANIFEST.in /web_common/ COPY web_common/web_common /web_common/web_common/ RUN hail-pip-install /web_common && rm -rf /web_common +RUN echo 'from hailtop import version; print(version());' | python3 From 8547426634da16dc4874a54e457f70bcffcec411 Mon Sep 17 00:00:00 2001 From: MIchael Franklin Date: Mon, 15 Mar 2021 10:02:14 +1100 Subject: [PATCH 3/5] Add version loading into service-java-run-base --- build.yaml | 3 +++ docker/Dockerfile.service-java-run-base | 3 +++ 2 files changed, 6 insertions(+) diff --git a/build.yaml b/build.yaml index dd78eb4b097..daa45e1c352 100644 --- a/build.yaml +++ b/build.yaml @@ -724,6 +724,9 @@ steps: publishAs: service-java-run-base dependsOn: - hail_ubuntu_image + inputs: + - from: /hail_version + to: /hail_version - kind: buildImage name: query_image dockerFile: query/Dockerfile diff --git a/docker/Dockerfile.service-java-run-base b/docker/Dockerfile.service-java-run-base index a9131df63c1..4ca7f6dd789 100644 --- a/docker/Dockerfile.service-java-run-base +++ b/docker/Dockerfile.service-java-run-base @@ -24,7 +24,10 @@ RUN hail-pip-install -r service-base-requirements.txt COPY hail/python/setup-hailtop.py /hailtop/setup.py COPY hail/python/hailtop /hailtop/hailtop/ +COPY /hail_version /hailtop/hailtop/hail_version +COPY hail/python/MANIFEST.in /hailtop/MANIFEST.in RUN hail-pip-install /hailtop && rm -rf /hailtop +RUN echo 'from hailtop import version; print(version());' | python3 COPY gear/setup.py /gear/setup.py COPY gear/gear /gear/gear/ From 62599b71259babb8ee7e15bf15db4baa00ed4c88 Mon Sep 17 00:00:00 2001 From: MIchael Franklin Date: Mon, 15 Mar 2021 10:22:59 +1100 Subject: [PATCH 4/5] Add make version to service-java-run-base --- docker/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Makefile b/docker/Makefile index 0111fbe8d9b..bbc7393be57 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -58,6 +58,8 @@ service-base: base-stmp .PHONY: service-java-run-base service-java-run-base: hail-ubuntu-stmp -docker pull $(SERVICE_JAVA_RUN_BASE_LATEST) + $(MAKE) -C ../hail python/hailtop/hail_version + cp ../hail/python/hailtop/hail_version /hail_version python3 ../ci/jinja2_render.py '{"hail_ubuntu_image":{"image":"hail-ubuntu"}}' Dockerfile.service-java-run-base Dockerfile.service-java-run-base.out [ "$(shell bash stat-permissions.sh Dockerfile.service-java-run-base.out)" = "644" ] docker build -t service-java-run-base -f Dockerfile.service-java-run-base.out --cache-from service-java-run-base,$(SERVICE_JAVA_RUN_BASE_LATEST),base,hail-ubuntu .. From bff3779015c3d36e60aece555d25e208b45d4779 Mon Sep 17 00:00:00 2001 From: MIchael Franklin Date: Mon, 15 Mar 2021 10:37:57 +1100 Subject: [PATCH 5/5] java-service-base should depend on copy_files --- build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/build.yaml b/build.yaml index daa45e1c352..43ecc016348 100644 --- a/build.yaml +++ b/build.yaml @@ -724,6 +724,7 @@ steps: publishAs: service-java-run-base dependsOn: - hail_ubuntu_image + - copy_files inputs: - from: /hail_version to: /hail_version