From 5ac7e7555c4cf6bde83d53a90ef85504f5e14794 Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 13:46:06 +0200 Subject: [PATCH 01/10] Try to add package version as block tag --- prefect_meemoo/config/blocks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/prefect_meemoo/config/blocks.py b/prefect_meemoo/config/blocks.py index 7043949..ff9cbc0 100644 --- a/prefect_meemoo/config/blocks.py +++ b/prefect_meemoo/config/blocks.py @@ -1,3 +1,5 @@ +import os + import pendulum from dateutil import parser from prefect.blocks.core import Block, SecretStr @@ -28,7 +30,10 @@ class LastRunConfig(Block): last_run_dict: dict = Field({}, description="Dictionary containing dates based on a certain value for a flow.") flow_name: str = Field(default=(...), description="The name of the flow.") name: str = Field(default=(...), description="The name of the deployment.") - _block_schema_capabilities = ["meemoo-prefect", "config"] + try: + _block_schema_capabilities = ["meemoo-prefect", "config", os.environ["BUILD_CONFIG_NAME"]] + except KeyError: + _block_schema_capabilities = ["meemoo-prefect", "config"] def get_last_run(self, format: str = "%Y-%m-%dT%H:%M:%S.%fZ", context: str = ""): if context: From f82d8e0ace4460baf35268ef649415c33c8ca8c1 Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 14:07:45 +0200 Subject: [PATCH 02/10] Fallback for package version --- prefect_meemoo/elasticsearch/credentials.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/prefect_meemoo/elasticsearch/credentials.py b/prefect_meemoo/elasticsearch/credentials.py index 68dd2e1..c85858b 100644 --- a/prefect_meemoo/elasticsearch/credentials.py +++ b/prefect_meemoo/elasticsearch/credentials.py @@ -1,3 +1,6 @@ +import os +from importlib.metadata import version + import urllib3 from elasticsearch import Elasticsearch from prefect.blocks.core import Block, SecretStr @@ -27,8 +30,10 @@ class ElasticsearchCredentials(Block): password: SecretStr = Field(default=(...), description="Elasticsearch password.") username: str = Field(default=(...), description="Elasticsearch username.") url: str = Field(default=(...), description="Elasticsearch URL.") - - _block_schema_capabilities = ["meemoo-prefect", "credentials"] + try: + _block_schema_capabilities = ["meemoo-prefect", "credentials", os.environ["BUILD_CONFIG_NAME"]] + except KeyError: + _block_schema_capabilities = ["meemoo-prefect", "credentials", version('prefect-meemoo')] def get_client(self) -> Elasticsearch: """ From 5e344ab02160c38732e9289e9b3a44ecab5ed51c Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 14:21:01 +0200 Subject: [PATCH 03/10] Add version to last run block --- prefect_meemoo/config/blocks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prefect_meemoo/config/blocks.py b/prefect_meemoo/config/blocks.py index ff9cbc0..2d7a11a 100644 --- a/prefect_meemoo/config/blocks.py +++ b/prefect_meemoo/config/blocks.py @@ -1,4 +1,5 @@ import os +from importlib.metadata import version import pendulum from dateutil import parser @@ -33,7 +34,7 @@ class LastRunConfig(Block): try: _block_schema_capabilities = ["meemoo-prefect", "config", os.environ["BUILD_CONFIG_NAME"]] except KeyError: - _block_schema_capabilities = ["meemoo-prefect", "config"] + _block_schema_capabilities = ["meemoo-prefect", "config", version('prefect-meemoo')] def get_last_run(self, format: str = "%Y-%m-%dT%H:%M:%S.%fZ", context: str = ""): if context: From 33950a3ac11b152d4dffe9766940f44f3ae4e200 Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 14:54:41 +0200 Subject: [PATCH 04/10] Add version to other blocks --- prefect_meemoo/config/blocks.py | 2 +- prefect_meemoo/elasticsearch/credentials.py | 2 +- prefect_meemoo/mediahaven/credentials.py | 8 +++++++- prefect_meemoo/triplydb/credentials.py | 8 +++++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/prefect_meemoo/config/blocks.py b/prefect_meemoo/config/blocks.py index 2d7a11a..208af4c 100644 --- a/prefect_meemoo/config/blocks.py +++ b/prefect_meemoo/config/blocks.py @@ -34,7 +34,7 @@ class LastRunConfig(Block): try: _block_schema_capabilities = ["meemoo-prefect", "config", os.environ["BUILD_CONFIG_NAME"]] except KeyError: - _block_schema_capabilities = ["meemoo-prefect", "config", version('prefect-meemoo')] + _block_schema_capabilities = ["meemoo-prefect", "config", "v"+ version('prefect-meemoo')] def get_last_run(self, format: str = "%Y-%m-%dT%H:%M:%S.%fZ", context: str = ""): if context: diff --git a/prefect_meemoo/elasticsearch/credentials.py b/prefect_meemoo/elasticsearch/credentials.py index c85858b..3501145 100644 --- a/prefect_meemoo/elasticsearch/credentials.py +++ b/prefect_meemoo/elasticsearch/credentials.py @@ -33,7 +33,7 @@ class ElasticsearchCredentials(Block): try: _block_schema_capabilities = ["meemoo-prefect", "credentials", os.environ["BUILD_CONFIG_NAME"]] except KeyError: - _block_schema_capabilities = ["meemoo-prefect", "credentials", version('prefect-meemoo')] + _block_schema_capabilities = ["meemoo-prefect", "credentials", "v"+ version('prefect-meemoo')] def get_client(self) -> Elasticsearch: """ diff --git a/prefect_meemoo/mediahaven/credentials.py b/prefect_meemoo/mediahaven/credentials.py index 5281c47..f550b32 100644 --- a/prefect_meemoo/mediahaven/credentials.py +++ b/prefect_meemoo/mediahaven/credentials.py @@ -1,3 +1,6 @@ +import os +from importlib.metadata import version + from mediahaven import MediaHaven from mediahaven.oauth2 import ROPCGrant from prefect.blocks.core import Block, SecretStr @@ -34,7 +37,10 @@ class MediahavenCredentials(Block): username: str = Field(default=(...), description="Mediahaven API username.") url: str = Field(default=(...), description="Mediahaven API URL.") - _block_schema_capabilities = ["meemoo-prefect", "credentials"] + try: + _block_schema_capabilities = ["meemoo-prefect", "credentials", os.environ["BUILD_CONFIG_NAME"]] + except KeyError: + _block_schema_capabilities = ["meemoo-prefect", "credentials", "v"+ version('prefect-meemoo')] def get_client(self) -> MediaHaven: """ diff --git a/prefect_meemoo/triplydb/credentials.py b/prefect_meemoo/triplydb/credentials.py index 451cb8d..b0d62b7 100644 --- a/prefect_meemoo/triplydb/credentials.py +++ b/prefect_meemoo/triplydb/credentials.py @@ -1,3 +1,6 @@ +import os +from importlib.metadata import version + from prefect.blocks.core import Block, SecretStr from pydantic import Field @@ -28,4 +31,7 @@ class TriplyDBCredentials(Block): host: str = Field(default=(...), description="TriplyDB HTTP host address.") gitlab_token: SecretStr = Field(default="", description="Gitlab token.") - _block_schema_capabilities = ["meemoo-prefect", "credentials"] \ No newline at end of file + try: + _block_schema_capabilities = ["meemoo-prefect", "credentials", os.environ["BUILD_CONFIG_NAME"]] + except KeyError: + _block_schema_capabilities = ["meemoo-prefect", "credentials", "v"+ version('prefect-meemoo')] \ No newline at end of file From 931a91e0d7c7aae729e2d301d607eb3acb83a904 Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 15:20:36 +0200 Subject: [PATCH 05/10] Update jenkinsfile var --- .openshift/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.openshift/Jenkinsfile b/.openshift/Jenkinsfile index 129ec0a..c403925 100644 --- a/.openshift/Jenkinsfile +++ b/.openshift/Jenkinsfile @@ -25,7 +25,7 @@ pipeline { env.IMAGE_TAG = sh(script: 'git describe --tags || echo latest', returnStdout: true) // The name used for the build config based on the image tag // Replace '.' with '_' as '.' is not allowed. - env.BUILD_CONFIG_NAME = sh(script: 'echo "${IMAGE_TAG}" | tr -d \'\\n\'', returnStdout: true) + env.BUILD_CONFIG_NAME = sh(script: 'echo "${IMAGE_TAG}" | tr -d \'\\n\' | sed -r "s/\\-//g"', returnStdout: true) env.DOCKER_IMAGE = sh(script: 'echo "${APP_NAME}:${GIT_SHORT_COMMIT}"') } } From d292fd5795a3e669afe59959d6ecdce2e35d11e7 Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 16:38:32 +0200 Subject: [PATCH 06/10] Fix test flow --- .openshift/Jenkinsfile | 2 +- tests/flows/flow_test_last_run_block.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.openshift/Jenkinsfile b/.openshift/Jenkinsfile index c403925..d7d1951 100644 --- a/.openshift/Jenkinsfile +++ b/.openshift/Jenkinsfile @@ -160,7 +160,7 @@ pipeline { script { def flows = readYaml file: '.openshift/flows.yaml' for (flow in flows.flows) { - runInPrefectContainer("prefect deployment build tests/flows/${flow.filename}:${flow.function} -n ${flow.name}-$DEPLOYMENT_ENV -ib docker-container/prefect-meemoo-test-flows-$DEPLOYMENT_ENV --skip-upload -q $PREFECT_QUEUE --apply") + runInPrefectContainer("prefect deployment build tests/flows/${flow.filename}:${flow.function} -n ${flow.name}-$DEPLOYMENT_ENV -ib docker-container/prefect-meemoo-test-flows-$DEPLOYMENT_ENV --skip-upload -q $PREFECT_QUEUE -t ${BUILD_CONFIG_NAME} --apply") } } } diff --git a/tests/flows/flow_test_last_run_block.py b/tests/flows/flow_test_last_run_block.py index 230eca5..a0078ea 100644 --- a/tests/flows/flow_test_last_run_block.py +++ b/tests/flows/flow_test_last_run_block.py @@ -2,7 +2,7 @@ from prefect import flow, get_run_logger, task from prefect.runtime import deployment, flow_run -from .prefect_meemoo.config.last_run import (add_last_run_with_context, +from prefect_meemoo.config.last_run import (add_last_run_with_context, get_last_run_config, save_last_run_config) From 1967ae972e7bef6b9a1ac257b8136adf293caaf5 Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 16:53:14 +0200 Subject: [PATCH 07/10] Update infra block --- tests/flows/infra_block.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/flows/infra_block.py b/tests/flows/infra_block.py index 26fa34a..d926d41 100644 --- a/tests/flows/infra_block.py +++ b/tests/flows/infra_block.py @@ -2,13 +2,16 @@ import argparse -def save_image(image_name, name, registry=None) -> None: +def save_image(image_name, name, registry=None, package_version=None) -> None: docker_registry = DockerRegistry.load(registry) docker_container = DockerContainer( image=image_name, auto_remove=True, image_registry=docker_registry, image_pull_policy="ALWAYS", + env={ + "BUILD_CONFIG_NAME": None + } ) docker_container.save(name=name, overwrite=True) @@ -22,5 +25,10 @@ def save_image(image_name, name, registry=None) -> None: required=False, help="Docker registry (the block which it contains), default docker.io", ) + parser.add_argument( + "--package_version", + required=False, + help="The version of the prefect-meemoo package", + ) args = parser.parse_args() save_image(args.image, args.name, args.registry) \ No newline at end of file From 70eca10ca163f3d175c1c5b2ec776114caedf556 Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 16:54:29 +0200 Subject: [PATCH 08/10] Update Jenkinsfile --- .openshift/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.openshift/Jenkinsfile b/.openshift/Jenkinsfile index d7d1951..ea7dc73 100644 --- a/.openshift/Jenkinsfile +++ b/.openshift/Jenkinsfile @@ -148,7 +148,7 @@ pipeline { } steps { script { - runInPrefectContainer("python tests/flows/infra_block.py --image $REGISTRY/prefect/prefect-meemoo-test-flows:$GIT_SHORT_COMMIT --name prefect-meemoo-test-flows-$DEPLOYMENT_ENV --registry docker-registry") + runInPrefectContainer("python tests/flows/infra_block.py --image $REGISTRY/prefect/prefect-meemoo-test-flows:$GIT_SHORT_COMMIT --name prefect-meemoo-test-flows-$DEPLOYMENT_ENV --registry docker-registry --package_version ${BUILD_CONFIG_NAME}") } } } From a6f33d74f6df199fc64741dd474ffb726d0b6d0f Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 16:59:08 +0200 Subject: [PATCH 09/10] Pass package version --- tests/flows/infra_block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flows/infra_block.py b/tests/flows/infra_block.py index d926d41..793ad5f 100644 --- a/tests/flows/infra_block.py +++ b/tests/flows/infra_block.py @@ -31,4 +31,4 @@ def save_image(image_name, name, registry=None, package_version=None) -> None: help="The version of the prefect-meemoo package", ) args = parser.parse_args() - save_image(args.image, args.name, args.registry) \ No newline at end of file + save_image(args.image, args.name, args.registry, args.package_version) \ No newline at end of file From 985a1c2cc0987e8631c58a6848f1d38f5c337d2c Mon Sep 17 00:00:00 2001 From: lennertvandevelde Date: Mon, 24 Jun 2024 17:03:17 +0200 Subject: [PATCH 10/10] Add package version to env --- tests/flows/infra_block.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flows/infra_block.py b/tests/flows/infra_block.py index 793ad5f..3a37e56 100644 --- a/tests/flows/infra_block.py +++ b/tests/flows/infra_block.py @@ -10,7 +10,7 @@ def save_image(image_name, name, registry=None, package_version=None) -> None: image_registry=docker_registry, image_pull_policy="ALWAYS", env={ - "BUILD_CONFIG_NAME": None + "BUILD_CONFIG_NAME": package_version } ) docker_container.save(name=name, overwrite=True)