diff --git a/docs/book/how-to/infrastructure-deployment/stack-deployment/README.md b/docs/book/how-to/infrastructure-deployment/stack-deployment/README.md index 4667b7e90ca..eb1e0b9f3d7 100644 --- a/docs/book/how-to/infrastructure-deployment/stack-deployment/README.md +++ b/docs/book/how-to/infrastructure-deployment/stack-deployment/README.md @@ -86,22 +86,27 @@ This docs section consists of information that makes it easier to provision, con Deploy a cloud stack with ZenML - Description of deploying a cloud stack with ZenML. + Deploy a cloud stack with ZenML ./deploy-a-cloud-stack.md Register a cloud stack - Description of registering a cloud stack. + Register a cloud stack ./register-a-cloud-stack.md Deploy a cloud stack with Terraform - Description of deploying a cloud stack with Terraform. + Deploy a cloud stack with Terraform ./deploy-a-cloud-stack-with-terraform.md + + Export and install stack requirements + Export and install stack requirements + ./export-stack-requirements.md + Reference secrets in stack configuration - Description of referencing secrets in stack configuration. + Reference secrets in stack configuration ./reference-secrets-in-stack-configuration.md @@ -109,11 +114,6 @@ This docs section consists of information that makes it easier to provision, con Creating your custom stack component solutions. ./implement-a-custom-stack-component.md - - Implement a custom integration - Description of implementing a custom integration. - ./implement-a-custom-integration.md - diff --git a/docs/book/how-to/infrastructure-deployment/stack-deployment/export-stack-requirements.md b/docs/book/how-to/infrastructure-deployment/stack-deployment/export-stack-requirements.md new file mode 100644 index 00000000000..1846530c61e --- /dev/null +++ b/docs/book/how-to/infrastructure-deployment/stack-deployment/export-stack-requirements.md @@ -0,0 +1,13 @@ +--- +description: Export stack requirements +--- + +You can get the `pip` requirements of your stack by running the `zenml stack export-requirements ` CLI command. + +To install those requirements, it's best to write them to a file and then install them like this: +```bash +zenml stack export-requirements --output-file stack_requirements.txt +pip install -r stack_requirements.txt +``` + +
ZenML Scarf
diff --git a/docs/book/toc.md b/docs/book/toc.md index 28b24626f03..349eba53ba3 100644 --- a/docs/book/toc.md +++ b/docs/book/toc.md @@ -155,6 +155,7 @@ * [Deploy a cloud stack with ZenML](how-to/infrastructure-deployment/stack-deployment/deploy-a-cloud-stack.md) * [Deploy a cloud stack with Terraform](how-to/infrastructure-deployment/stack-deployment/deploy-a-cloud-stack-with-terraform.md) * [Register a cloud stack](how-to/infrastructure-deployment/stack-deployment/register-a-cloud-stack.md) + * [Export and install stack requirements](how-to/infrastructure-deployment/stack-deployment/export-stack-requirements.md) * [Reference secrets in stack configuration](how-to/infrastructure-deployment/stack-deployment/reference-secrets-in-stack-configuration.md) * [Implement a custom stack component](how-to/infrastructure-deployment/stack-deployment/implement-a-custom-stack-component.md) * [Customize Docker builds](how-to/infrastructure-deployment/customize-docker-builds/README.md) diff --git a/docs/book/user-guide/production-guide/understand-stacks.md b/docs/book/user-guide/production-guide/understand-stacks.md index 1dbe8435dea..6b18ea2513c 100644 --- a/docs/book/user-guide/production-guide/understand-stacks.md +++ b/docs/book/user-guide/production-guide/understand-stacks.md @@ -14,10 +14,12 @@ A `stack` is the configuration of tools and infrastructure that your pipelines c As visualized in the diagram above, there are two separate domains that are connected through ZenML. The left side shows the code domain. The user's Python code is translated into a ZenML pipeline. On the right side, you can see the infrastructure domain, in this case, an instance of the `default` stack. By separating these two domains, it is easy to switch the environment that the pipeline runs on without making any changes in the code. It also allows domain experts to write code/configure infrastructure without worrying about the other domain. +{% hint style="info" %} +You can get the `pip` requirements of your stack by running the `zenml stack export-requirements ` CLI command. +{% endhint %} + ### The `default` stack -{% tabs %} -{% tab title="CLI" %} `zenml stack describe` lets you find out details about your active stack: ```bash @@ -50,8 +52,6 @@ Stack 'default' with id '...' is owned by user default and is 'private'. {% hint style="info" %} As you can see a stack can be **active** on your **client**. This simply means that any pipeline you run will be using the **active stack** as its environment. {% endhint %} -{% endtab %} -{% endtabs %} ## Components of a stack @@ -61,8 +61,6 @@ As you can see in the section above, a stack consists of multiple components. Al The **orchestrator** is responsible for executing the pipeline code. In the simplest case, this will be a simple Python thread on your machine. Let's explore this default orchestrator. -{% tabs %} -{% tab title="CLI" %} `zenml orchestrator list` lets you see all orchestrators that are registered in your zenml deployment. ```bash @@ -72,15 +70,11 @@ The **orchestrator** is responsible for executing the pipeline code. In the simp ┃ 👉 │ default │ ... │ local │ ➖ │ default ┃ ┗━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━┷━━━━━━━━━┛ ``` -{% endtab %} -{% endtabs %} ### Artifact store The **artifact store** is responsible for persisting the step outputs. As we learned in the previous section, the step outputs are not passed along in memory, rather the outputs of each step are stored in the **artifact store** and then loaded from there when the next step needs them. By default this will also be on your own machine: -{% tabs %} -{% tab title="CLI" %} `zenml artifact-store list` lets you see all artifact stores that are registered in your zenml deployment. ```bash @@ -90,8 +84,6 @@ The **artifact store** is responsible for persisting the step outputs. As we lea ┃ 👉 │ default │ ... │ local │ ➖ │ default ┃ ┗━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━┷━━━━━━━━┷━━━━━━━━┷━━━━━━━━━┛ ``` -{% endtab %} -{% endtabs %} ### Other stack components @@ -105,8 +97,6 @@ Just to illustrate how to interact with stacks, let's create an alternate local ### Create an artifact store -{% tabs %} -{% tab title="CLI" %} ```bash zenml artifact-store register my_artifact_store --flavor=local ``` @@ -131,15 +121,11 @@ To see the new artifact store that you just registered, just run: ```bash zenml artifact-store describe my_artifact_store ``` -{% endtab %} -{% endtabs %} ### Create a local stack With the artifact store created, we can now create a new stack with this artifact store. -{% tabs %} -{% tab title="CLI" %} ```bash zenml stack register a_new_local_stack -o default -a my_artifact_store ``` @@ -177,8 +163,6 @@ Which will give you an output like this: 'a_new_local_stack' stack Stack 'a_new_local_stack' with id '...' is owned by user default and is 'private'. ``` -{% endtab %} -{% endtabs %} ### Switch stacks with our VS Code extension diff --git a/src/zenml/cli/__init__.py b/src/zenml/cli/__init__.py index d6dfcf21502..3803611527a 100644 --- a/src/zenml/cli/__init__.py +++ b/src/zenml/cli/__init__.py @@ -1525,6 +1525,13 @@ zenml stack rename STACK_NAME NEW_STACK_NAME ``` +If you would like to export the requirements of your stack, you can +use the command: + +```bash +zenml stack export-requirements +``` + If you want to copy a stack component, run the following command: ```bash zenml STACK_COMPONENT copy SOURCE_COMPONENT_NAME TARGET_COMPONENT_NAME diff --git a/src/zenml/cli/stack.py b/src/zenml/cli/stack.py index 6962f46a870..1781a759134 100644 --- a/src/zenml/cli/stack.py +++ b/src/zenml/cli/stack.py @@ -75,6 +75,7 @@ from zenml.service_connectors.service_connector_utils import ( get_resources_options_from_resource_model_for_full_stack, ) +from zenml.utils import requirements_utils from zenml.utils.dashboard_utils import get_component_url, get_stack_url from zenml.utils.yaml_utils import read_yaml, write_yaml @@ -1944,3 +1945,77 @@ def query_region( service_connector_index=service_connector_index, service_connector_resource_id=service_connector_resource_id, ) + + +@stack.command( + name="export-requirements", help="Export the stack requirements." +) +@click.argument( + "stack_name_or_id", + type=click.STRING, + required=False, +) +@click.option( + "--output-file", + "-o", + "output_file", + type=str, + required=False, + help="File to which to export the stack requirements. If not " + "provided, the requirements will be printed to stdout instead.", +) +@click.option( + "--overwrite", + "-ov", + "overwrite", + type=bool, + required=False, + is_flag=True, + help="Overwrite the output file if it already exists. This option is " + "only valid if the output file is provided.", +) +def export_requirements( + stack_name_or_id: Optional[str] = None, + output_file: Optional[str] = None, + overwrite: bool = False, +) -> None: + """Exports stack requirements so they can be installed using pip. + + Args: + stack_name_or_id: Stack name or ID. If not given, the active stack will + be used. + output_file: Optional path to the requirements output file. + overwrite: Overwrite the output file if it already exists. This option + is only valid if the output file is provided. + """ + try: + stack_model: "StackResponse" = Client().get_stack( + name_id_or_prefix=stack_name_or_id + ) + except KeyError as err: + cli_utils.error(str(err)) + + requirements, _ = requirements_utils.get_requirements_for_stack( + stack_model + ) + + if not requirements: + cli_utils.declare(f"Stack `{stack_model.name}` has no requirements.") + return + + if output_file: + try: + with open(output_file, "x") as f: + f.write("\n".join(requirements)) + except FileExistsError: + if overwrite or cli_utils.confirmation( + "A file already exists at the specified path. " + "Would you like to overwrite it?" + ): + with open(output_file, "w") as f: + f.write("\n".join(requirements)) + cli_utils.declare( + f"Requirements for stack `{stack_model.name}` exported to {output_file}." + ) + else: + click.echo(" ".join(requirements), nl=False) diff --git a/src/zenml/utils/requirements_utils.py b/src/zenml/utils/requirements_utils.py new file mode 100644 index 00000000000..5e3e57bc16b --- /dev/null +++ b/src/zenml/utils/requirements_utils.py @@ -0,0 +1,71 @@ +# Copyright (c) ZenML GmbH 2024. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. +"""Requirement utils.""" + +from typing import TYPE_CHECKING, List, Set, Tuple + +from zenml.integrations.utils import get_integration_for_module + +if TYPE_CHECKING: + from zenml.models import ComponentResponse, StackResponse + + +def get_requirements_for_stack( + stack: "StackResponse", +) -> Tuple[List[str], List[str]]: + """Get requirements for a stack model. + + Args: + stack: The stack for which to get the requirements. + + Returns: + Tuple of PyPI and APT requirements of the stack. + """ + pypi_requirements: Set[str] = set() + apt_packages: Set[str] = set() + + for component_list in stack.components.values(): + assert len(component_list) == 1 + component = component_list[0] + ( + component_pypi_requirements, + component_apt_packages, + ) = get_requirements_for_component(component=component) + pypi_requirements = pypi_requirements.union( + component_pypi_requirements + ) + apt_packages = apt_packages.union(component_apt_packages) + + return sorted(pypi_requirements), sorted(apt_packages) + + +def get_requirements_for_component( + component: "ComponentResponse", +) -> Tuple[List[str], List[str]]: + """Get requirements for a component model. + + Args: + component: The component for which to get the requirements. + + Returns: + Tuple of PyPI and APT requirements of the component. + """ + integration = get_integration_for_module( + module_name=component.flavor.source + ) + + if integration: + return integration.get_requirements(), integration.APT_PACKAGES + else: + return [], [] diff --git a/src/zenml/zen_server/template_execution/utils.py b/src/zenml/zen_server/template_execution/utils.py index f537fcc8ce4..34831d89029 100644 --- a/src/zenml/zen_server/template_execution/utils.py +++ b/src/zenml/zen_server/template_execution/utils.py @@ -3,7 +3,7 @@ import copy import hashlib import sys -from typing import Any, Dict, List, Optional, Set, Tuple +from typing import Any, Dict, List, Optional from uuid import UUID from fastapi import BackgroundTasks @@ -22,11 +22,9 @@ ENV_ZENML_ACTIVE_WORKSPACE_ID, ) from zenml.enums import ExecutionStatus, StackComponentType, StoreType -from zenml.integrations.utils import get_integration_for_module from zenml.logger import get_logger from zenml.models import ( CodeReferenceRequest, - ComponentResponse, FlavorFilter, PipelineDeploymentRequest, PipelineDeploymentResponse, @@ -43,7 +41,7 @@ validate_stack_is_runnable_from_server, ) from zenml.stack.flavor import Flavor -from zenml.utils import dict_utils, settings_utils +from zenml.utils import dict_utils, requirements_utils, settings_utils from zenml.zen_server.auth import AuthContext from zenml.zen_server.template_execution.runner_entrypoint_configuration import ( RunnerEntrypointConfiguration, @@ -151,8 +149,8 @@ def run_template( assert placeholder_run def _task() -> None: - pypi_requirements, apt_packages = get_requirements_for_stack( - stack=stack + pypi_requirements, apt_packages = ( + requirements_utils.get_requirements_for_stack(stack=stack) ) if build.python_version: @@ -266,59 +264,6 @@ def ensure_async_orchestrator( ) -def get_requirements_for_stack( - stack: StackResponse, -) -> Tuple[List[str], List[str]]: - """Get requirements for a stack model. - - Args: - stack: The stack for which to get the requirements. - - Returns: - Tuple of PyPI and APT requirements of the stack. - """ - pypi_requirements: Set[str] = set() - apt_packages: Set[str] = set() - - for component_list in stack.components.values(): - assert len(component_list) == 1 - component = component_list[0] - ( - component_pypi_requirements, - component_apt_packages, - ) = get_requirements_for_component(component=component) - pypi_requirements = pypi_requirements.union( - component_pypi_requirements - ) - apt_packages = apt_packages.union(component_apt_packages) - - return sorted(pypi_requirements), sorted(apt_packages) - - -def get_requirements_for_component( - component: ComponentResponse, -) -> Tuple[List[str], List[str]]: - """Get requirements for a component model. - - Args: - component: The component for which to get the requirements. - - Returns: - Tuple of PyPI and APT requirements of the component. - """ - flavors = zen_store().list_flavors( - FlavorFilter(name=component.flavor_name, type=component.type) - ) - assert len(flavors) == 1 - flavor_source = flavors[0].source - integration = get_integration_for_module(module_name=flavor_source) - - if integration: - return integration.get_requirements(), integration.APT_PACKAGES - else: - return [], [] - - def generate_image_hash(dockerfile: str) -> str: """Generate a hash of the Dockerfile.