From 66296eac1648941ccb73c8d0c86d65f85c654568 Mon Sep 17 00:00:00 2001 From: Alex Ganose Date: Wed, 8 Nov 2023 13:29:43 +0000 Subject: [PATCH 01/26] Fix JobStoreDocument validator --- src/jobflow/core/job.py | 2 +- src/jobflow/core/schemas.py | 34 ++++++++++ src/jobflow/core/store.py | 2 +- src/jobflow/schemas/job_output_schema.py | 63 ------------------- ...t_job_output_schema.py => test_schemas.py} | 4 +- 5 files changed, 38 insertions(+), 67 deletions(-) create mode 100644 src/jobflow/core/schemas.py delete mode 100644 src/jobflow/schemas/job_output_schema.py rename tests/core/{test_job_output_schema.py => test_schemas.py} (95%) diff --git a/src/jobflow/core/job.py b/src/jobflow/core/job.py index e7ba11a3..f11e1917 100644 --- a/src/jobflow/core/job.py +++ b/src/jobflow/core/job.py @@ -10,7 +10,6 @@ from monty.json import MSONable, jsanitize from jobflow.core.reference import OnMissing, OutputReference -from jobflow.schemas.job_output_schema import JobStoreDocument from jobflow.utils.uuid import suuid if typing.TYPE_CHECKING: @@ -560,6 +559,7 @@ def run(self, store: jobflow.JobStore) -> Response: from jobflow import CURRENT_JOB from jobflow.core.flow import get_flow + from jobflow.core.schemas import JobStoreDocument index_str = f", {self.index}" if self.index != 1 else "" logger.info(f"Starting job - {self.name} ({self.uuid}{index_str})") diff --git a/src/jobflow/core/schemas.py b/src/jobflow/core/schemas.py new file mode 100644 index 00000000..09c84107 --- /dev/null +++ b/src/jobflow/core/schemas.py @@ -0,0 +1,34 @@ +"""A Pydantic model for Jobstore document.""" + +from typing import Any + +from pydantic import BaseModel, Field + + +class JobStoreDocument(BaseModel): + """A Pydantic model for Jobstore document.""" + + uuid: str = Field( + None, description="An unique identifier for the job. Generated automatically." + ) + index: int = Field( + None, + description="The index of the job (number of times the job has been replaced).", + ) + output: Any = Field( + None, + description="This is a reference to the future job output.", + ) + completed_at: str = Field(None, description="The time the job was completed.") + metadata: dict = Field( + None, + description="Metadata information supplied by the user.", + ) + hosts: list[str] = Field( + None, + description="The list of UUIDs of the hosts containing the job.", + ) + name: str = Field( + None, + description="The name of the job.", + ) diff --git a/src/jobflow/core/store.py b/src/jobflow/core/store.py index 923cbb46..45da2246 100644 --- a/src/jobflow/core/store.py +++ b/src/jobflow/core/store.py @@ -18,7 +18,7 @@ from maggma.core import Sort - from jobflow.schemas.job_output_schema import JobStoreDocument + from jobflow.core.schemas import JobStoreDocument obj_type = Union[str, Enum, type[MSONable], list[Union[Enum, str, type[MSONable]]]] save_type = Optional[dict[str, obj_type]] diff --git a/src/jobflow/schemas/job_output_schema.py b/src/jobflow/schemas/job_output_schema.py deleted file mode 100644 index da08d98b..00000000 --- a/src/jobflow/schemas/job_output_schema.py +++ /dev/null @@ -1,63 +0,0 @@ -"""A Pydantic model for Jobstore document.""" - -from typing import Generic, TypeVar - -from monty.json import MontyDecoder -from pydantic import BaseModel, Field, field_validator - -T = TypeVar("T") - - -class JobStoreDocument(BaseModel, Generic[T]): - """A Pydantic model for Jobstore document.""" - - uuid: str = Field( - None, description="An unique identifier for the job. Generated automatically." - ) - index: int = Field( - None, - description="The index of the job (number of times the job has been replaced).", - ) - output: T = Field( - None, - description="This is a reference to the future job output.", - ) - completed_at: str = Field(None, description="The time the job was completed.") - metadata: dict = Field( - None, - description="Metadeta information supplied by the user.", - ) - hosts: list[str] = Field( - None, - description="The list of UUIDs of the hosts containing the job.", - ) - name: str = Field( - None, - description="The name of the job.", - ) - - @field_validator("output", mode="before") - @classmethod - def reserialize_output(cls, v): - """ - Pre-validator for the 'output' field. - - This method checks if the input 'v' is a dictionary with specific keys - ('@module' and '@class'). If these keys are present, it reprocesses - the input dictionary using MontyDecoder to deserialize it. - - Parameters - ---------- - cls : Type[JobStoreDocument] - The class this validator is applied to. - v : Any - The input value to validate. - - Returns - ------- - Any - The validated and potentially deserialized value. - """ - if isinstance(v, dict) and "@module" in v and "@class" in v: - v = MontyDecoder().process_decoded(v) - return v diff --git a/tests/core/test_job_output_schema.py b/tests/core/test_schemas.py similarity index 95% rename from tests/core/test_job_output_schema.py rename to tests/core/test_schemas.py index 2db2c330..84af8ace 100644 --- a/tests/core/test_job_output_schema.py +++ b/tests/core/test_schemas.py @@ -5,7 +5,7 @@ @pytest.fixture() def sample_data(): - from jobflow.schemas.job_output_schema import JobStoreDocument + from jobflow.core.schemas import JobStoreDocument return JobStoreDocument( uuid="abc123", @@ -33,7 +33,7 @@ def test_job_store_document_model(sample_data): def test_job_store_update(memory_jobstore, sample_data): # Storing document as a JobStoreDocument - from jobflow.schemas.job_output_schema import JobStoreDocument + from jobflow.core.schemas import JobStoreDocument d = { "index": 1, From 63ca3653f83fd37b49f2942177304be5cfc7a265 Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Mon, 13 Nov 2023 08:12:47 -0800 Subject: [PATCH 02/26] JOSS: Add missing `,` to paper Addressed first bullet point by reviewer: https://github.com/openjournals/joss-reviews/issues/5995#issuecomment-1808396589 --- paper/paper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper/paper.md b/paper/paper.md index fa043a8d..fb46383c 100644 --- a/paper/paper.md +++ b/paper/paper.md @@ -117,7 +117,7 @@ responses = run_locally(flow) ## Dynamic Workflows -Beyond the typical acyclic graph of jobs, Jobflow fully supports dynamic workflows where the precise number of jobs is unknown until runtime. This is a particularly common requirement in chemistry and materials science workflows and is made possible through the use of a `Response` object that controls the flow execution. For instance, the example below is a `Flow` that will add two numbers (`1 + 2`), construct a list of random length containing the prior result (e.g. `[3, 3, 3]`), and then add an integer to each element of the list (`[3 + 10, 3 + 10, 3 + 10]`). The `Response(replace=Flow(jobs))` syntax tells Jobflow to replace the current `Job` with a (sub)workflow after the `Job` completes. +Beyond the typical acyclic graph of jobs, Jobflow fully supports dynamic workflows where the precise number of jobs is unknown until runtime. This is a particularly common requirement in chemistry and materials science workflows and is made possible through the use of a `Response` object that controls the flow execution. For instance, the example below is a `Flow` that will add two numbers (`1 + 2`), construct a list of random length containing the prior result (e.g., `[3, 3, 3]`), and then add an integer to each element of the list (`[3 + 10, 3 + 10, 3 + 10]`). The `Response(replace=Flow(jobs))` syntax tells Jobflow to replace the current `Job` with a (sub)workflow after the `Job` completes. ![](figure2.png) From 3b47876783bce6122ced6ce14505d20ff6024792 Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Mon, 13 Nov 2023 08:18:39 -0800 Subject: [PATCH 03/26] JOSS: Fix references to include all coauthors and remove "others" This was a leftover issue from fetching references from Google Scholar. Addresses https://github.com/openjournals/joss-reviews/issues/5995#issuecomment-1808396589 --- paper/refs.bib | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/paper/refs.bib b/paper/refs.bib index 47f15be7..66324680 100644 --- a/paper/refs.bib +++ b/paper/refs.bib @@ -11,7 +11,7 @@ @misc{quacc } @article{materialsproject, title={Commentary: The Materials Project: A materials genome approach to accelerating materials innovation}, - author={Jain, Anubhav and Ong, Shyue Ping and Hautier, Geoffroy and Chen, Wei and Richards, William Davidson and Dacek, Stephen and Cholia, Shreyas and Gunter, Dan and Skinner, David and Ceder, Gerbrand and others}, + author={Jain, Anubhav and Ong, Shyue Ping and Hautier, Geoffroy and Chen, Wei and Richards, William Davidson and Dacek, Stephen and Cholia, Shreyas and Gunter, Dan and Skinner, David and Ceder, Gerbrand and Persson, Kristin A}, journal={APL materials}, volume={1}, number={1}, @@ -57,7 +57,7 @@ @misc{nptools } @article{fireworks, title={FireWorks: a dynamic workflow system designed for high-throughput applications}, -author={Jain, Anubhav and Ong, Shyue Ping and Chen, Wei and Medasani, Bharat and Qu, Xiaohui and Kocher, Michael and Brafman, Miriam and Petretto, Guido and Rignanese, Gian-Marco and Hautier, Geoffroy and others}, +author={Jain, Anubhav and Ong, Shyue Ping and Chen, Wei and Medasani, Bharat and Qu, Xiaohui and Kocher, Michael and Brafman, Miriam and Petretto, Guido and Rignanese, Gian-Marco and Hautier, Geoffroy and Gunter, Daniel and Persson, Kristin A}, journal={Concurrency and Computation: Practice and Experience}, volume={27}, number={17}, @@ -68,7 +68,7 @@ @article{fireworks } @article{da2023workflows, title={Workflows Community Summit 2022: A Roadmap Revolution}, -author={da Silva, Rafael Ferreira and Badia, Rosa M and Bala, Venkat and Bard, Debbie and Bremer, Peer-Timo and Buckley, Ian and Caino-Lores, Silvina and Chard, Kyle and Goble, Carole and Jha, Shantenu and others}, +author={Da Silva, Rafael Ferreira and Badia, Rosa M. and Bala, Venkat and Bard, Debbie and Bremer, Peer-Timo and Buckley, Ian and Caino-Lores, Silvina and Chard, Kyle and Goble, Carole and Jha, Shantenu and Katz, Daniel S. and Laney, Daniel and Parashar, Manish and Suter, Frederic and Tyler, Nick and Uram, Thomas and Altintas, Ilkay and Andersson, Stefan and Arndt, William and Aznar, Juan and Bader, Jonathan and Balis, Bartosz and Blanton, Chris and Braghetto, Kelly Rosa and Brodutch, Aharon and Brunk, Paul and Casanova, Henri and Lierta, Alba Cervera and Chigu, Justin and Coleman, Taina and Collier, Nick and Colonnelli, Iacopo and Coppens, Frederik and Crusoe, Michael and Cunningham, Will and De Paula Kinoshita, Bruno and Di Tommaso, Paolo and Doutriaux, Charles and Downton, Matthew and Elwasif, Wael and Enders, Bjoern and Erdmann, Chris and Fahringer, Thomas and Figueiredo, Ludmilla and Filgueira, Rosa and Foltin, Martin and Fouilloux, Anne and Gadelha, Luiz and Gallo, Andy and Saez, Artur Garcia and Garijo, Daniel and Gerlach, Roman and Grant, Ryan and Grayson, Samuel and Grubel, Patricia and Gustafsson, Johan and Hayot-Sasson, Valerie and Hernandez, Oscar and Hilbrich, Marcus and Justine, AnnMary and Laflotte, Ian and Lehmann, Fabian and Luckow, Andre and Luettgau, Jakob and Maheshwari, Ketan and Matsuda, Motohiko and Medic, Doriana and Mendygral, Pete and Michalewicz, Marek and Nonaka, Jorji and Pawlik, Maciej and Pottier, Loic and Pouchard, Line and Putz, Mathias and Radha, Santosh Kumar and Ramakrishnan, Lavanya and Ristov, Sashko and Romano, Paul and Rosendo, Daniel and Ruefenacht, Martin and Rycerz, Katarzyna and Saurabh, Nishant and Savchenko, Volodymyr and Schulz, Martin and Simpson, Christine and Sirvent, Raul and Skluzacek, Tyler and Soiland-Reyes, Stian and Souza, Renan and Sukumar, Sreenivas Rangan and Sun, Ziheng and Sussman, Alan and Thain, Douglas and Titov, Mikhail and Tovar, Benjamin and Tripathy, Aalap and Turilli, Matteo and Tuznik, Bartosz and Van Dam, Hubertus and Vivas, Aurelio and Ward, Logan and Widener, Patrick and Wilkinson, Sean and Zawalska, Justyna and Zulfiqar, Mahnoor}, journal={arXiv preprint arXiv:2304.00019}, year={2023} } @@ -88,7 +88,7 @@ @misc{wflowsystems } @inproceedings{al2021exaworks, title={Exaworks: Workflows for exascale}, -author={Al-Saadi, Aymen and Ahn, Dong H and Babuji, Yadu and Chard, Kyle and Corbett, James and Hategan, Mihael and Herbein, Stephen and Jha, Shantenu and Laney, Daniel and Merzky, Andre and others}, +author={Al-Saadi, Aymen and Ahn, Dong H and Babuji, Yadu and Chard, Kyle and Corbett, James and Hategan, Mihael and Herbein, Stephen and Jha, Shantenu and Laney, Daniel and Merzky, Andre and Munson, Todd and Salim, Michael and Titov, Mikhail and Uram, Thomas D and Wozniak, Justin M}, booktitle={2021 IEEE Workshop on Workflows in Support of Large-Scale Science (WORKS)}, pages={50--57}, year={2021}, @@ -97,7 +97,7 @@ @inproceedings{al2021exaworks } @inproceedings{babuji2019parsl, title={Parsl: Pervasive parallel programming in python}, -author={Babuji, Yadu and Woodard, Anna and Li, Zhuozhao and Katz, Daniel S and Clifford, Ben and Kumar, Rohan and Lacinski, Lukasz and Chard, Ryan and Wozniak, Justin M and Foster, Ian and others}, +author={Babuji, Yadu and Woodard, Anna and Li, Zhuozhao and Katz, Daniel S and Clifford, Ben and Kumar, Rohan and Lacinski, Lukasz and Chard, Ryan and Wozniak, Justin M and Foster, Ian and Wilde, Michael and Chard, Kyle}, booktitle={Proceedings of the 28th International Symposium on High-Performance Parallel and Distributed Computing}, pages={25--36}, year={2019} From 41da97d1b91034271d87a513302d03eb38687013 Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Mon, 13 Nov 2023 08:30:16 -0800 Subject: [PATCH 04/26] JOSS: Add more complete Zenodo reference info Addresses https://github.com/openjournals/joss-reviews/issues/5995#issuecomment-1808396589. Namely, I used a minimal format for the Zenodo references. Now, I have used the one auto-generated by Zenodo. One downside of this approach is that the author list includes all contributors to the repo, no matter how large or small their contributions are. I see two ways to go about this: 1) Just accept it and move on. 2) Keep the rest of the metadata but remove the `author` field. @utf, any thoughts? --- paper/refs.bib | 53 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 10 deletions(-) diff --git a/paper/refs.bib b/paper/refs.bib index 47f15be7..415f1d87 100644 --- a/paper/refs.bib +++ b/paper/refs.bib @@ -3,11 +3,14 @@ @misc{atomate2 year = {2023}, url = {https://github.com/materialsproject/atomate2} } -@misc{quacc, -doi = {10.5281/zenodo.7720998}, -title = {Quacc -- The Quantum Accelerator}, -year = {2023}, -url = {https://github.com/Quantum-Accelerators/quacc} +@software{quacc, + author = {Rosen, Andrew S}, + title = {quacc – The Quantum Accelerator}, + year = 2023, + publisher = {Zenodo}, + version = {v0.4.0}, + doi = {10.5281/zenodo.10091610}, + url = {https://doi.org/10.5281/zenodo.10091610} } @article{materialsproject, title={Commentary: The Materials Project: A materials genome approach to accelerating materials innovation}, @@ -112,11 +115,41 @@ @misc{redun year = {2023}, url={https://github.com/insitro/redun} } -@misc{covalent, -title={Covalent}, -year = {2023}, -doi={10.5281/zenodo.5903364}, -url={https://github.com/AgnostiqHQ/covalent} +@software{covalent, + author = {Cunningham, Will + Esquivel, Alejandro and + Jao, Casey and + Hasan, Faiyaz and + Bala, Venkat and + Sanand, Sankalp and + Venkatesh, Prasanna and + Tandon, Madhur and + Emmanuel Ochia, Okechukwu and + Rosen, Andrew S and + dwelsch-esi and + jkanem and + Aravind and + HaimHorowitzAgnostiq and + Li, Ruihao and + Neagle, Scott Wyman and + valkostadinov and + Ghukasyan, Ara and + Rao, Poojith U and + Dutta, Sayandip and + WingCode and + Hughes, Anna and + RaviPsiog and + Udayan and + Akalanka and + Obasi, Amara and + Singh, Divyanshu and + FilipBolt}, + title = {Covalent}, + year = 2023, + publisher = {Zenodo}, + version = {v0.228.0-rc.0}, + doi = {10.5281/zenodo.8369670}, + url = {https://doi.org/10.5281/zenodo.8369670} } @misc{maggma, title={Maggma}, From a8013bf088b844f31678b38a06ec1c7a60747c0d Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Mon, 13 Nov 2023 08:33:51 -0800 Subject: [PATCH 05/26] Update refs.bib --- paper/refs.bib | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/paper/refs.bib b/paper/refs.bib index 415f1d87..1aa20668 100644 --- a/paper/refs.bib +++ b/paper/refs.bib @@ -8,9 +8,8 @@ @software{quacc title = {quacc – The Quantum Accelerator}, year = 2023, publisher = {Zenodo}, - version = {v0.4.0}, - doi = {10.5281/zenodo.10091610}, - url = {https://doi.org/10.5281/zenodo.10091610} + doi = {10.5281/zenodo.7720998}, + url = {https://doi.org/10.5281/zenodo.7720998} } @article{materialsproject, title={Commentary: The Materials Project: A materials genome approach to accelerating materials innovation}, From 6bf46f67f4f3cc8d51e479af55e1996a392e8a27 Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Mon, 13 Nov 2023 08:34:18 -0800 Subject: [PATCH 06/26] Update refs.bib --- paper/refs.bib | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/paper/refs.bib b/paper/refs.bib index 1aa20668..f2f09a28 100644 --- a/paper/refs.bib +++ b/paper/refs.bib @@ -146,9 +146,8 @@ @software{covalent title = {Covalent}, year = 2023, publisher = {Zenodo}, - version = {v0.228.0-rc.0}, - doi = {10.5281/zenodo.8369670}, - url = {https://doi.org/10.5281/zenodo.8369670} + doi = {10.5281/zenodo.5903364}, + url = {https://doi.org/10.5281/zenodo.5903364} } @misc{maggma, title={Maggma}, From 66a67fdd8058e4171d8d9203174eedb6eee949ef Mon Sep 17 00:00:00 2001 From: "Andrew S. Rosen" Date: Mon, 13 Nov 2023 08:35:04 -0800 Subject: [PATCH 07/26] Update refs.bib --- paper/refs.bib | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paper/refs.bib b/paper/refs.bib index f2f09a28..b5bd243a 100644 --- a/paper/refs.bib +++ b/paper/refs.bib @@ -6,7 +6,7 @@ @misc{atomate2 @software{quacc, author = {Rosen, Andrew S}, title = {quacc – The Quantum Accelerator}, - year = 2023, + year = {2023}, publisher = {Zenodo}, doi = {10.5281/zenodo.7720998}, url = {https://doi.org/10.5281/zenodo.7720998} @@ -144,7 +144,7 @@ @software{covalent Singh, Divyanshu and FilipBolt}, title = {Covalent}, - year = 2023, + year = {2023}, publisher = {Zenodo}, doi = {10.5281/zenodo.5903364}, url = {https://doi.org/10.5281/zenodo.5903364} From 2c6bc41c7a51f0793a447ece3fe0079c37e918e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 23:25:20 +0000 Subject: [PATCH 08/26] Bump pydantic from 2.4.2 to 2.5.0 (#483) Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.4.2 to 2.5.0. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.4.2...v2.5.0) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 631f98c5..bf3fd3a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ strict = [ "moto==4.2.7", "networkx==3.2.1", "pydantic-settings==2.0.3", - "pydantic==2.4.2", + "pydantic==2.5.0", "pydash==7.0.6", "pydot==1.4.2", "typing-extensions==4.8.0", From 2938c013619ada50f11fbafbaa7f7d2611b502e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 23:25:27 +0000 Subject: [PATCH 09/26] Bump moto from 4.2.7 to 4.2.8 (#482) Bumps [moto](https://github.com/getmoto/moto) from 4.2.7 to 4.2.8. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/4.2.7...4.2.8) --- updated-dependencies: - dependency-name: moto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bf3fd3a2..f7e43c40 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ docs = [ "sphinx==7.2.6", ] dev = ["pre-commit>=2.12.1"] -tests = ["moto==4.2.7", "pytest-cov==4.1.0", "pytest==7.4.3"] +tests = ["moto==4.2.8", "pytest-cov==4.1.0", "pytest==7.4.3"] vis = ["matplotlib", "pydot"] fireworks = ["FireWorks"] strict = [ @@ -55,7 +55,7 @@ strict = [ "maggma==0.57.5", "matplotlib==3.8.1", "monty==2023.11.3", - "moto==4.2.7", + "moto==4.2.8", "networkx==3.2.1", "pydantic-settings==2.0.3", "pydantic==2.5.0", From 61bbaa449b631bce24fdcaa52f7d13d30ba4899b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 23:25:44 +0000 Subject: [PATCH 10/26] Bump maggma from 0.57.5 to 0.57.8 (#484) Bumps [maggma](https://github.com/materialsproject/maggma) from 0.57.5 to 0.57.8. - [Release notes](https://github.com/materialsproject/maggma/releases) - [Changelog](https://github.com/materialsproject/maggma/blob/main/docs/CHANGELOG.md) - [Commits](https://github.com/materialsproject/maggma/compare/v0.57.5...v0.57.8) --- updated-dependencies: - dependency-name: maggma dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index f7e43c40..0e1731b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ fireworks = ["FireWorks"] strict = [ "FireWorks==2.0.3", "PyYAML==6.0.1", - "maggma==0.57.5", + "maggma==0.57.8", "matplotlib==3.8.1", "monty==2023.11.3", "moto==4.2.8", From 36577151ace24ccb5f3c78f178af5c28235dc833 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Wed, 15 Nov 2023 17:21:13 -0800 Subject: [PATCH 11/26] run_locally() make root_dir a keyword so users don't have to change directory to control where flows run --- src/jobflow/managers/local.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/jobflow/managers/local.py b/src/jobflow/managers/local.py index f9c15bf1..5c1fb9fb 100644 --- a/src/jobflow/managers/local.py +++ b/src/jobflow/managers/local.py @@ -6,8 +6,9 @@ import typing if typing.TYPE_CHECKING: - import jobflow + from pathlib import Path + import jobflow logger = logging.getLogger(__name__) @@ -17,6 +18,7 @@ def run_locally( log: bool = True, store: jobflow.JobStore = None, create_folders: bool = False, + root_dir: str | Path | None = None, ensure_success: bool = False, allow_external_references: bool = False, ) -> dict[str, dict[int, jobflow.Response]]: @@ -25,25 +27,29 @@ def run_locally( Parameters ---------- - flow + flow : Flow | Job | list[Job] A job or flow. - log + log : bool Whether to print log messages. - store + store : JobStore A job store. If a job store is not specified then :obj:`JobflowSettings.JOB_STORE` will be used. By default this is a maggma ``MemoryStore`` but can be customised by setting the jobflow configuration file. - create_folders + create_folders : bool Whether to run each job in a new folder. - ensure_success + root_dir : str | Path | None + The root directory to run the jobs in or where to create new subfolders if + ``create_folders`` is True. If None then the current working + directory will be used. + ensure_success : bool Raise an error if the flow was not executed successfully. - allow_external_references + allow_external_references : bool If False all the references to other outputs should be from other Jobs of the Flow. Returns ------- - Dict[str, Dict[int, Response]] + dict[str, dict[int, Response]] The responses of the jobs, as a dict of ``{uuid: {index: response}}``. """ from collections import defaultdict @@ -60,6 +66,9 @@ def run_locally( if store is None: store = SETTINGS.JOB_STORE + root_dir = Path.cwd() if root_dir is None else Path(root_dir).resolve() + root_dir.mkdir(exist_ok=True) + store.connect() if log: @@ -72,8 +81,6 @@ def run_locally( responses: dict[str, dict[int, jobflow.Response]] = defaultdict(dict) stop_jobflow = False - root_dir = Path.cwd() - def _run_job(job: jobflow.Job, parents): nonlocal stop_jobflow From bf36d98a5460d7c13550395c5d8fbae04b19ef3d Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Wed, 15 Nov 2023 17:21:36 -0800 Subject: [PATCH 12/26] test run_locally with root_dir --- tests/managers/test_local.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/managers/test_local.py b/tests/managers/test_local.py index 94c1ce0e..bc4e6008 100644 --- a/tests/managers/test_local.py +++ b/tests/managers/test_local.py @@ -62,6 +62,14 @@ def test_simple_flow(memory_jobstore, clean_dir, simple_flow, capsys): folders = list(Path(".").glob("job_*/")) assert len(folders) == 1 + # run with folders and root_dir + responses = run_locally( + flow, store=memory_jobstore, create_folders=True, root_dir="test" + ) + assert responses[uuid][1].output == "12345_end" + folders = list(Path("test").glob("job_*/")) + assert len(folders) == 1 + def test_connected_flow(memory_jobstore, clean_dir, connected_flow): from jobflow import run_locally From a8d45878533cf096ed424ba217fb9ad45941897f Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Wed, 15 Nov 2023 18:11:39 -0800 Subject: [PATCH 13/26] improve test --- src/jobflow/core/reference.py | 6 +++++- tests/managers/test_local.py | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/jobflow/core/reference.py b/src/jobflow/core/reference.py index 0700389a..be5874c8 100644 --- a/src/jobflow/core/reference.py +++ b/src/jobflow/core/reference.py @@ -177,7 +177,11 @@ def resolve( for attr_type, attr in self.attributes: # i means index else use attribute access - data = data[attr] if attr_type == "i" else getattr(data, attr) + data = ( + data[attr] + if attr_type == "i" or isinstance(data, dict) + else getattr(data, attr) + ) return data diff --git a/tests/managers/test_local.py b/tests/managers/test_local.py index bc4e6008..bc85be12 100644 --- a/tests/managers/test_local.py +++ b/tests/managers/test_local.py @@ -63,11 +63,12 @@ def test_simple_flow(memory_jobstore, clean_dir, simple_flow, capsys): assert len(folders) == 1 # run with folders and root_dir + assert Path(root_dir := "test").exists() is False responses = run_locally( - flow, store=memory_jobstore, create_folders=True, root_dir="test" + flow, store=memory_jobstore, create_folders=True, root_dir=root_dir ) assert responses[uuid][1].output == "12345_end" - folders = list(Path("test").glob("job_*/")) + folders = list(Path(root_dir).glob("job_*/")) assert len(folders) == 1 From d6cd86d9298c56b73fbdc65197d7f3650e5813d3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 00:01:55 +0000 Subject: [PATCH 14/26] Bump maggma from 0.57.8 to 0.57.10 (#488) Bumps [maggma](https://github.com/materialsproject/maggma) from 0.57.8 to 0.57.10. - [Release notes](https://github.com/materialsproject/maggma/releases) - [Changelog](https://github.com/materialsproject/maggma/blob/main/docs/CHANGELOG.md) - [Commits](https://github.com/materialsproject/maggma/compare/v0.57.8...v0.57.10) --- updated-dependencies: - dependency-name: maggma dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0e1731b8..1c5c9958 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ fireworks = ["FireWorks"] strict = [ "FireWorks==2.0.3", "PyYAML==6.0.1", - "maggma==0.57.8", + "maggma==0.57.10", "matplotlib==3.8.1", "monty==2023.11.3", "moto==4.2.8", From 4ceba93c5ec39c7b1bc1261d2854f75655259ef2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 00:02:02 +0000 Subject: [PATCH 15/26] Bump moto from 4.2.8 to 4.2.9 (#489) Bumps [moto](https://github.com/getmoto/moto) from 4.2.8 to 4.2.9. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/4.2.8...4.2.9) --- updated-dependencies: - dependency-name: moto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1c5c9958..b12fc5c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ docs = [ "sphinx==7.2.6", ] dev = ["pre-commit>=2.12.1"] -tests = ["moto==4.2.8", "pytest-cov==4.1.0", "pytest==7.4.3"] +tests = ["moto==4.2.9", "pytest-cov==4.1.0", "pytest==7.4.3"] vis = ["matplotlib", "pydot"] fireworks = ["FireWorks"] strict = [ @@ -55,7 +55,7 @@ strict = [ "maggma==0.57.10", "matplotlib==3.8.1", "monty==2023.11.3", - "moto==4.2.8", + "moto==4.2.9", "networkx==3.2.1", "pydantic-settings==2.0.3", "pydantic==2.5.0", From 33b303c4c633d44bf4063d8fa0466be2d0d0d513 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 00:02:22 +0000 Subject: [PATCH 16/26] Bump pydantic from 2.5.0 to 2.5.1 (#490) Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.5.0 to 2.5.1. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/main/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.5.0...v2.5.1) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b12fc5c6..4b8ceed5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ strict = [ "moto==4.2.9", "networkx==3.2.1", "pydantic-settings==2.0.3", - "pydantic==2.5.0", + "pydantic==2.5.1", "pydash==7.0.6", "pydot==1.4.2", "typing-extensions==4.8.0", From 0b2db72143bb23e52ff2ee9a5f4a52743b7cefdd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Nov 2023 00:04:40 +0000 Subject: [PATCH 17/26] Bump matplotlib from 3.8.1 to 3.8.2 (#491) Bumps [matplotlib](https://github.com/matplotlib/matplotlib) from 3.8.1 to 3.8.2. - [Release notes](https://github.com/matplotlib/matplotlib/releases) - [Commits](https://github.com/matplotlib/matplotlib/compare/v3.8.1...v3.8.2) --- updated-dependencies: - dependency-name: matplotlib dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 4b8ceed5..d6cf8cc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ strict = [ "FireWorks==2.0.3", "PyYAML==6.0.1", "maggma==0.57.10", - "matplotlib==3.8.1", + "matplotlib==3.8.2", "monty==2023.11.3", "moto==4.2.9", "networkx==3.2.1", From c87139f5221ab277a8e3661fa429f47fffcf093b Mon Sep 17 00:00:00 2001 From: JaGeo Date: Thu, 23 Nov 2023 23:48:49 +0100 Subject: [PATCH 18/26] Add fibonacci example to tutorial --- docs/tutorials/5-dynamic-flows.ipynb | 295 ++++++++++++++++++++------- 1 file changed, 221 insertions(+), 74 deletions(-) diff --git a/docs/tutorials/5-dynamic-flows.ipynb b/docs/tutorials/5-dynamic-flows.ipynb index b99861ed..9673cd21 100644 --- a/docs/tutorials/5-dynamic-flows.ipynb +++ b/docs/tutorials/5-dynamic-flows.ipynb @@ -39,11 +39,15 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "id": "78348798", "metadata": { "nbsphinx": "hidden", - "tags": [] + "tags": [], + "ExecuteTime": { + "end_time": "2023-11-23T22:46:57.984085360Z", + "start_time": "2023-11-23T22:46:57.166923399Z" + } }, "outputs": [], "source": [ @@ -54,35 +58,40 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 3, "id": "964a8c44", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-23T22:46:58.036098609Z", + "start_time": "2023-11-23T22:46:57.208641695Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2023-06-08 09:55:05,762 INFO Started executing jobs locally\n", - "2023-06-08 09:55:05,873 INFO Starting job - make_list (68b72ca9-00ae-430f-a61c-fbea22bbf0aa)\n", - "2023-06-08 09:55:05,874 INFO Finished job - make_list (68b72ca9-00ae-430f-a61c-fbea22bbf0aa)\n", - "2023-06-08 09:55:05,874 INFO Starting job - add_distributed (01821a62-3ee2-47f0-a6ca-fd373ea0ea09)\n", - "2023-06-08 09:55:05,875 INFO Finished job - add_distributed (01821a62-3ee2-47f0-a6ca-fd373ea0ea09)\n", - "2023-06-08 09:55:05,876 INFO Starting job - add (fbc49130-2bbe-49b0-a190-48815c03a0b0)\n", - "2023-06-08 09:55:05,876 INFO Finished job - add (fbc49130-2bbe-49b0-a190-48815c03a0b0)\n", - "2023-06-08 09:55:05,877 INFO Starting job - add (ca3636e3-50f7-4ec8-a5e7-66bc4e14b901)\n", - "2023-06-08 09:55:05,877 INFO Finished job - add (ca3636e3-50f7-4ec8-a5e7-66bc4e14b901)\n", - "2023-06-08 09:55:05,877 INFO Starting job - add (28dbe7c4-b29b-46c4-9d0c-2aee7334b999)\n", - "2023-06-08 09:55:05,878 INFO Finished job - add (28dbe7c4-b29b-46c4-9d0c-2aee7334b999)\n", - "2023-06-08 09:55:05,878 INFO Starting job - add (1bffefd1-4edc-43d5-b99f-b0833fca2b8d)\n", - "2023-06-08 09:55:05,879 INFO Finished job - add (1bffefd1-4edc-43d5-b99f-b0833fca2b8d)\n", - "2023-06-08 09:55:05,879 INFO Finished executing jobs locally\n" + "2023-11-23 23:46:57,177 INFO Started executing jobs locally\n", + "2023-11-23 23:46:57,179 INFO Starting job - make_list (16fc2fd0-c9c7-420d-b0e1-3e6b5802c744)\n", + "2023-11-23 23:46:57,182 INFO Finished job - make_list (16fc2fd0-c9c7-420d-b0e1-3e6b5802c744)\n", + "2023-11-23 23:46:57,183 INFO Starting job - add_distributed (536e3c78-7d59-43ed-9f4f-72ce9b3793ef)\n", + "2023-11-23 23:46:57,191 INFO Finished job - add_distributed (536e3c78-7d59-43ed-9f4f-72ce9b3793ef)\n", + "2023-11-23 23:46:57,193 INFO Starting job - add (44007968-8394-442f-ba96-d52dc30a80ce)\n", + "2023-11-23 23:46:57,195 INFO Finished job - add (44007968-8394-442f-ba96-d52dc30a80ce)\n", + "2023-11-23 23:46:57,196 INFO Starting job - add (1af57ed7-3594-4180-b550-787c5f029d9f)\n", + "2023-11-23 23:46:57,198 INFO Finished job - add (1af57ed7-3594-4180-b550-787c5f029d9f)\n", + "2023-11-23 23:46:57,199 INFO Starting job - add (f7ea9158-40f8-4439-9d18-b6b6fb3e9289)\n", + "2023-11-23 23:46:57,201 INFO Finished job - add (f7ea9158-40f8-4439-9d18-b6b6fb3e9289)\n", + "2023-11-23 23:46:57,202 INFO Starting job - add (72c752d3-208a-45ce-bbb6-625e4debed73)\n", + "2023-11-23 23:46:57,204 INFO Finished job - add (72c752d3-208a-45ce-bbb6-625e4debed73)\n", + "2023-11-23 23:46:57,205 INFO Finished executing jobs locally\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ - "/Users/alexmganose/dev/src/jobflow/src/jobflow/utils/graph.py:53: UserWarning: Some jobs are not connected, their ordering may be random\n", + "/home/jgeorge/miniconda3/envs/AddJobflowTutorial/lib/python3.10/site-packages/jobflow/utils/graph.py:49: UserWarning: Some jobs are not connected, their ordering may be random\n", " warnings.warn(\"Some jobs are not connected, their ordering may be random\")\n" ] } @@ -118,20 +127,29 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "id": "82e368e6", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-23T22:46:58.037599975Z", + "start_time": "2023-11-23T22:46:57.233955503Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "68b72ca9-00ae-430f-a61c-fbea22bbf0aa -> {1: Response(output=[2, 2, 2, 2], detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "01821a62-3ee2-47f0-a6ca-fd373ea0ea09 -> {1: Response(output=None, detour=None, addition=None, replace=, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "fbc49130-2bbe-49b0-a190-48815c03a0b0 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "ca3636e3-50f7-4ec8-a5e7-66bc4e14b901 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "28dbe7c4-b29b-46c4-9d0c-2aee7334b999 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "1bffefd1-4edc-43d5-b99f-b0833fca2b8d -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" + "16fc2fd0-c9c7-420d-b0e1-3e6b5802c744 -> {1: Response(output=[2, 2, 2, 2], detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "536e3c78-7d59-43ed-9f4f-72ce9b3793ef -> {1: Response(output=None, detour=None, addition=None, replace=Flow(name='Flow', uuid='1d51c5df-174a-41c8-b017-98494c8411f1')\n", + "1. Job(name='add', uuid='44007968-8394-442f-ba96-d52dc30a80ce')\n", + "2. Job(name='add', uuid='1af57ed7-3594-4180-b550-787c5f029d9f')\n", + "3. Job(name='add', uuid='f7ea9158-40f8-4439-9d18-b6b6fb3e9289')\n", + "4. Job(name='add', uuid='72c752d3-208a-45ce-bbb6-625e4debed73'), stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "44007968-8394-442f-ba96-d52dc30a80ce -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "1af57ed7-3594-4180-b550-787c5f029d9f -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "f7ea9158-40f8-4439-9d18-b6b6fb3e9289 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "72c752d3-208a-45ce-bbb6-625e4debed73 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" ] } ], @@ -171,22 +189,27 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "id": "81870e55", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-23T22:46:58.041273240Z", + "start_time": "2023-11-23T22:46:57.406089273Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2023-06-08 09:55:07,374 INFO Started executing jobs locally\n", - "2023-06-08 09:55:07,377 INFO Starting job - add (8270f7cc-e12a-457b-adca-a196ee62ae85)\n", - "2023-06-08 09:55:07,379 INFO Finished job - add (8270f7cc-e12a-457b-adca-a196ee62ae85)\n", - "2023-06-08 09:55:07,380 INFO Starting job - add_with_logic (fd549146-aabb-4e4b-b79a-ca6b6fd2c172)\n", - "2023-06-08 09:55:07,382 INFO Finished job - add_with_logic (fd549146-aabb-4e4b-b79a-ca6b6fd2c172)\n", - "2023-06-08 09:55:07,384 INFO Starting job - add (9db2454f-4f1d-4d08-8497-1239cad74d0c)\n", - "2023-06-08 09:55:07,384 INFO Finished job - add (9db2454f-4f1d-4d08-8497-1239cad74d0c)\n", - "2023-06-08 09:55:07,385 INFO Finished executing jobs locally\n" + "2023-11-23 23:46:57,406 INFO Started executing jobs locally\n", + "2023-11-23 23:46:57,408 INFO Starting job - add (34ba86a3-99cd-461e-bc17-511fdf6e08ec)\n", + "2023-11-23 23:46:57,411 INFO Finished job - add (34ba86a3-99cd-461e-bc17-511fdf6e08ec)\n", + "2023-11-23 23:46:57,412 INFO Starting job - add_with_logic (349afb69-d732-44b8-b770-2500f44511f9)\n", + "2023-11-23 23:46:57,421 INFO Finished job - add_with_logic (349afb69-d732-44b8-b770-2500f44511f9)\n", + "2023-11-23 23:46:57,422 INFO Starting job - add (64e623da-cf31-485e-9ed1-1f996a955868)\n", + "2023-11-23 23:46:57,424 INFO Finished job - add (64e623da-cf31-485e-9ed1-1f996a955868)\n", + "2023-11-23 23:46:57,425 INFO Finished executing jobs locally\n" ] } ], @@ -209,17 +232,23 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 6, "id": "fdc68eb5", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-23T22:46:58.046206634Z", + "start_time": "2023-11-23T22:46:57.448864584Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "8270f7cc-e12a-457b-adca-a196ee62ae85 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "fd549146-aabb-4e4b-b79a-ca6b6fd2c172 -> {1: Response(output=None, detour=None, addition=, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "9db2454f-4f1d-4d08-8497-1239cad74d0c -> {1: Response(output=5, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" + "34ba86a3-99cd-461e-bc17-511fdf6e08ec -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "349afb69-d732-44b8-b770-2500f44511f9 -> {1: Response(output=None, detour=None, addition=Flow(name='Flow', uuid='20581dd8-b00e-424b-94ad-7f2b2f4a240b')\n", + "1. Job(name='add', uuid='64e623da-cf31-485e-9ed1-1f996a955868'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "64e623da-cf31-485e-9ed1-1f996a955868 -> {1: Response(output=5, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" ] } ], @@ -238,20 +267,25 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "id": "27569bd7", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-23T22:46:58.090608163Z", + "start_time": "2023-11-23T22:46:57.481314817Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2023-06-08 09:55:08,838 INFO Started executing jobs locally\n", - "2023-06-08 09:55:08,841 INFO Starting job - add (ec551667-1500-45cc-b61a-2361a18b99fd)\n", - "2023-06-08 09:55:08,843 INFO Finished job - add (ec551667-1500-45cc-b61a-2361a18b99fd)\n", - "2023-06-08 09:55:08,843 INFO Starting job - add_with_logic (e7d3fb53-fa56-45bd-9951-45b70028ada7)\n", - "2023-06-08 09:55:08,846 INFO Finished job - add_with_logic (e7d3fb53-fa56-45bd-9951-45b70028ada7)\n", - "2023-06-08 09:55:08,847 INFO Finished executing jobs locally\n" + "2023-11-23 23:46:57,479 INFO Started executing jobs locally\n", + "2023-11-23 23:46:57,481 INFO Starting job - add (0ae20220-b69c-4e6b-827a-2f046b00d504)\n", + "2023-11-23 23:46:57,484 INFO Finished job - add (0ae20220-b69c-4e6b-827a-2f046b00d504)\n", + "2023-11-23 23:46:57,485 INFO Starting job - add_with_logic (145dea09-be6c-47fc-ac46-c9a6908323e7)\n", + "2023-11-23 23:46:57,490 INFO Finished job - add_with_logic (145dea09-be6c-47fc-ac46-c9a6908323e7)\n", + "2023-11-23 23:46:57,491 INFO Finished executing jobs locally\n" ] } ], @@ -274,16 +308,21 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "id": "b8d0c011", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-23T22:46:58.092445839Z", + "start_time": "2023-11-23T22:46:57.524568954Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "ec551667-1500-45cc-b61a-2361a18b99fd -> {1: Response(output=21, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "e7d3fb53-fa56-45bd-9951-45b70028ada7 -> {1: Response(output=None, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" + "0ae20220-b69c-4e6b-827a-2f046b00d504 -> {1: Response(output=21, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "145dea09-be6c-47fc-ac46-c9a6908323e7 -> {1: Response(output=None, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" ] } ], @@ -300,6 +339,108 @@ "Now, we see that the `Response(addition)` does not launch a new job.\n" ] }, + { + "cell_type": "markdown", + "source": [ + "In this way, one can also compute the Fibonacci numbers:" + ], + "metadata": { + "collapsed": false + } + }, + { + "cell_type": "code", + "execution_count": 9, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2023-11-23 23:46:57,625 INFO Started executing jobs locally\n", + "2023-11-23 23:46:57,626 INFO Starting job - fibonacci (81d7944a-bd5a-4edd-a67f-36d2312e506b)\n", + "2023-11-23 23:46:57,630 INFO Finished job - fibonacci (81d7944a-bd5a-4edd-a67f-36d2312e506b)\n", + "2023-11-23 23:46:57,631 INFO Starting job - fibonacci (3219e168-b719-409a-8dbd-94370c03e162)\n", + "2023-11-23 23:46:57,634 INFO Finished job - fibonacci (3219e168-b719-409a-8dbd-94370c03e162)\n", + "2023-11-23 23:46:57,636 INFO Starting job - fibonacci (98308b42-817b-4ee5-bf68-fefcf1bd0de3)\n", + "2023-11-23 23:46:57,639 INFO Finished job - fibonacci (98308b42-817b-4ee5-bf68-fefcf1bd0de3)\n", + "2023-11-23 23:46:57,640 INFO Starting job - fibonacci (57c73c23-9fbf-4f08-8e0f-7698027b632b)\n", + "2023-11-23 23:46:57,643 INFO Finished job - fibonacci (57c73c23-9fbf-4f08-8e0f-7698027b632b)\n", + "2023-11-23 23:46:57,644 INFO Starting job - fibonacci (33c81f02-e87b-4848-8780-a09de20041b1)\n", + "2023-11-23 23:46:57,647 INFO Finished job - fibonacci (33c81f02-e87b-4848-8780-a09de20041b1)\n", + "2023-11-23 23:46:57,648 INFO Starting job - fibonacci (690e4d45-9e9b-43d8-8865-2085cc0feee2)\n", + "2023-11-23 23:46:57,650 INFO Finished job - fibonacci (690e4d45-9e9b-43d8-8865-2085cc0feee2)\n", + "2023-11-23 23:46:57,651 INFO Starting job - fibonacci (8142cc72-a4e3-4884-a446-98f0bea98569)\n", + "2023-11-23 23:46:57,655 INFO Finished job - fibonacci (8142cc72-a4e3-4884-a446-98f0bea98569)\n", + "2023-11-23 23:46:57,656 INFO Starting job - fibonacci (c2b4ec59-5088-4daf-9dba-34a2a3fbe56e)\n", + "2023-11-23 23:46:57,659 INFO Finished job - fibonacci (c2b4ec59-5088-4daf-9dba-34a2a3fbe56e)\n", + "2023-11-23 23:46:57,660 INFO Starting job - fibonacci (2db909a8-14df-4bd1-89a8-d188948382f1)\n", + "2023-11-23 23:46:57,663 INFO Finished job - fibonacci (2db909a8-14df-4bd1-89a8-d188948382f1)\n", + "2023-11-23 23:46:57,664 INFO Starting job - fibonacci (273a46cc-4762-495e-a1d7-e4e26203f122)\n", + "2023-11-23 23:46:57,666 INFO Finished job - fibonacci (273a46cc-4762-495e-a1d7-e4e26203f122)\n", + "2023-11-23 23:46:57,667 INFO Starting job - fibonacci (3fe6a08d-74f9-406f-a219-fdd68f1f6ec9)\n", + "2023-11-23 23:46:57,670 INFO Finished job - fibonacci (3fe6a08d-74f9-406f-a219-fdd68f1f6ec9)\n", + "2023-11-23 23:46:57,671 INFO Starting job - fibonacci (5c44497a-4497-4bf2-a0ee-172abf1251eb)\n", + "2023-11-23 23:46:57,674 INFO Finished job - fibonacci (5c44497a-4497-4bf2-a0ee-172abf1251eb)\n", + "2023-11-23 23:46:57,675 INFO Starting job - fibonacci (1b21a0be-8b79-42ad-a13b-c43c7077d637)\n", + "2023-11-23 23:46:57,678 INFO Finished job - fibonacci (1b21a0be-8b79-42ad-a13b-c43c7077d637)\n", + "2023-11-23 23:46:57,679 INFO Starting job - fibonacci (a6af5fcc-2a3b-4a40-8620-28edadb64b9f)\n", + "2023-11-23 23:46:57,682 INFO Finished job - fibonacci (a6af5fcc-2a3b-4a40-8620-28edadb64b9f)\n", + "2023-11-23 23:46:57,683 INFO Starting job - fibonacci (552ad89c-3e1f-4385-8eb5-01a761a6f9f1)\n", + "2023-11-23 23:46:57,686 INFO Finished job - fibonacci (552ad89c-3e1f-4385-8eb5-01a761a6f9f1)\n", + "2023-11-23 23:46:57,687 INFO Finished executing jobs locally\n", + "{'81d7944a-bd5a-4edd-a67f-36d2312e506b': {1: Response(output=2, detour=None, addition=Flow(name='Flow', uuid='b497505f-e73a-4324-99b4-2923d76bed6e')\n", + "1. Job(name='fibonacci', uuid='3219e168-b719-409a-8dbd-94370c03e162'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '3219e168-b719-409a-8dbd-94370c03e162': {1: Response(output=3, detour=None, addition=Flow(name='Flow', uuid='985aad5f-060f-4841-a2d0-edcdb4a4526d')\n", + "1. Job(name='fibonacci', uuid='98308b42-817b-4ee5-bf68-fefcf1bd0de3'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '98308b42-817b-4ee5-bf68-fefcf1bd0de3': {1: Response(output=5, detour=None, addition=Flow(name='Flow', uuid='076cc046-7ffe-40fe-a14d-5c97d9e92b40')\n", + "1. Job(name='fibonacci', uuid='57c73c23-9fbf-4f08-8e0f-7698027b632b'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '57c73c23-9fbf-4f08-8e0f-7698027b632b': {1: Response(output=8, detour=None, addition=Flow(name='Flow', uuid='38b57dd4-ddb1-425c-8153-5e2c195942da')\n", + "1. Job(name='fibonacci', uuid='33c81f02-e87b-4848-8780-a09de20041b1'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '33c81f02-e87b-4848-8780-a09de20041b1': {1: Response(output=13, detour=None, addition=Flow(name='Flow', uuid='057e0e16-62d3-461e-a13b-9d86c5f72780')\n", + "1. Job(name='fibonacci', uuid='690e4d45-9e9b-43d8-8865-2085cc0feee2'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '690e4d45-9e9b-43d8-8865-2085cc0feee2': {1: Response(output=21, detour=None, addition=Flow(name='Flow', uuid='a2f5bb27-5539-48be-a706-04af3756eedf')\n", + "1. Job(name='fibonacci', uuid='8142cc72-a4e3-4884-a446-98f0bea98569'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '8142cc72-a4e3-4884-a446-98f0bea98569': {1: Response(output=34, detour=None, addition=Flow(name='Flow', uuid='c0b854aa-aae5-4277-b7e1-45416e65384e')\n", + "1. Job(name='fibonacci', uuid='c2b4ec59-5088-4daf-9dba-34a2a3fbe56e'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, 'c2b4ec59-5088-4daf-9dba-34a2a3fbe56e': {1: Response(output=55, detour=None, addition=Flow(name='Flow', uuid='f31f638f-4147-4677-b8f5-72e44d9b12f9')\n", + "1. Job(name='fibonacci', uuid='2db909a8-14df-4bd1-89a8-d188948382f1'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '2db909a8-14df-4bd1-89a8-d188948382f1': {1: Response(output=89, detour=None, addition=Flow(name='Flow', uuid='5f5383a3-7b2d-4e86-98ca-66636006e6a3')\n", + "1. Job(name='fibonacci', uuid='273a46cc-4762-495e-a1d7-e4e26203f122'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '273a46cc-4762-495e-a1d7-e4e26203f122': {1: Response(output=144, detour=None, addition=Flow(name='Flow', uuid='4711a431-8976-4e7c-be1c-cb4962aab4a7')\n", + "1. Job(name='fibonacci', uuid='3fe6a08d-74f9-406f-a219-fdd68f1f6ec9'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '3fe6a08d-74f9-406f-a219-fdd68f1f6ec9': {1: Response(output=233, detour=None, addition=Flow(name='Flow', uuid='0dc7cb69-5797-4880-a8b7-c54f99490322')\n", + "1. Job(name='fibonacci', uuid='5c44497a-4497-4bf2-a0ee-172abf1251eb'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '5c44497a-4497-4bf2-a0ee-172abf1251eb': {1: Response(output=377, detour=None, addition=Flow(name='Flow', uuid='2469da06-bb02-4597-925c-df154d34115d')\n", + "1. Job(name='fibonacci', uuid='1b21a0be-8b79-42ad-a13b-c43c7077d637'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '1b21a0be-8b79-42ad-a13b-c43c7077d637': {1: Response(output=610, detour=None, addition=Flow(name='Flow', uuid='c92c422d-372c-45bd-9159-1ef9243c8f7f')\n", + "1. Job(name='fibonacci', uuid='a6af5fcc-2a3b-4a40-8620-28edadb64b9f'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, 'a6af5fcc-2a3b-4a40-8620-28edadb64b9f': {1: Response(output=987, detour=None, addition=Flow(name='Flow', uuid='4e7d136c-a125-4c08-b660-247bea293d64')\n", + "1. Job(name='fibonacci', uuid='552ad89c-3e1f-4385-8eb5-01a761a6f9f1'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '552ad89c-3e1f-4385-8eb5-01a761a6f9f1': {1: Response(output=1597, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}}\n" + ] + } + ], + "source": [ + "\"\"\"A dynamic workflow that calculates the Fibonacci sequence.\"\"\"\n", + "from jobflow import Response, job, run_locally\n", + "\n", + "\n", + "@job\n", + "def fibonacci(smaller, larger, stop_point=1000):\n", + " \"\"\"Calculate the next number in the Fibonacci sequence.\n", + "\n", + " If the number is larger than stop_point, the job will stop the workflow\n", + " execution, otherwise, a new job will be submitted to calculate the next number.\n", + " \"\"\"\n", + " total = smaller + larger\n", + "\n", + " if total > stop_point:\n", + " return total\n", + "\n", + " new_job = fibonacci(larger, total, stop_point=stop_point)\n", + " return Response(output=total, addition=new_job)\n", + "\n", + "\n", + "fibonacci_job = fibonacci(1, 1)\n", + "\n", + "# run the job; responses will contain the output from all jobs\n", + "responses = run_locally(fibonacci_job)\n", + "print(responses)\n" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "end_time": "2023-11-23T22:46:58.136876182Z", + "start_time": "2023-11-23T22:46:57.625858812Z" + } + } + }, { "cell_type": "markdown", "id": "5c0122e3", @@ -318,22 +459,27 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 10, "id": "fee35a97", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-23T22:46:58.142624609Z", + "start_time": "2023-11-23T22:46:57.774129081Z" + } + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2023-06-08 09:55:10,850 INFO Started executing jobs locally\n", - "2023-06-08 09:55:10,853 INFO Starting job - add (d5020d71-4bbc-4307-8fa5-c65d0d9b9b9a)\n", - "2023-06-08 09:55:10,855 INFO Finished job - add (d5020d71-4bbc-4307-8fa5-c65d0d9b9b9a)\n", - "2023-06-08 09:55:10,856 INFO Starting job - add_with_logic (8786b3c4-5697-4ed8-9f46-fd0c89e4c374)\n", - "2023-06-08 09:55:10,858 INFO Finished job - add_with_logic (8786b3c4-5697-4ed8-9f46-fd0c89e4c374)\n", - "2023-06-08 09:55:10,859 INFO Starting job - add (c33d72c2-f396-4f2c-898e-c667c0913dcb)\n", - "2023-06-08 09:55:10,859 INFO Finished job - add (c33d72c2-f396-4f2c-898e-c667c0913dcb)\n", - "2023-06-08 09:55:10,860 INFO Finished executing jobs locally\n" + "2023-11-23 23:46:57,774 INFO Started executing jobs locally\n", + "2023-11-23 23:46:57,775 INFO Starting job - add (55465994-efcd-4835-b92e-f7ea36765268)\n", + "2023-11-23 23:46:57,778 INFO Finished job - add (55465994-efcd-4835-b92e-f7ea36765268)\n", + "2023-11-23 23:46:57,779 INFO Starting job - add_with_logic (d6cf8229-dbcc-422c-8504-177c433f48ac)\n", + "2023-11-23 23:46:57,786 INFO Finished job - add_with_logic (d6cf8229-dbcc-422c-8504-177c433f48ac)\n", + "2023-11-23 23:46:57,788 INFO Starting job - add (996e6af8-c56f-4779-8135-00493dbf9298)\n", + "2023-11-23 23:46:57,790 INFO Finished job - add (996e6af8-c56f-4779-8135-00493dbf9298)\n", + "2023-11-23 23:46:57,791 INFO Finished executing jobs locally\n" ] } ], @@ -356,19 +502,20 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "id": "b3b7e8ba", - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2023-11-23T22:46:58.143531724Z", + "start_time": "2023-11-23T22:46:57.816557390Z" + } + }, "outputs": [ { "data": { - "text/plain": [ - "{'d5020d71-4bbc-4307-8fa5-c65d0d9b9b9a': {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)},\n", - " '8786b3c4-5697-4ed8-9f46-fd0c89e4c374': {1: Response(output=None, detour=, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)},\n", - " 'c33d72c2-f396-4f2c-898e-c667c0913dcb': {1: Response(output=5, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}}" - ] + "text/plain": "{'55465994-efcd-4835-b92e-f7ea36765268': {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)},\n 'd6cf8229-dbcc-422c-8504-177c433f48ac': {1: Response(output=None, detour=Flow(name='Flow', uuid='98311314-8087-4787-b2ea-c89478ff2999')\n 1. Job(name='add', uuid='996e6af8-c56f-4779-8135-00493dbf9298'), addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)},\n '996e6af8-c56f-4779-8135-00493dbf9298': {1: Response(output=5, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}}" }, - "execution_count": 9, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -388,9 +535,9 @@ ], "metadata": { "kernelspec": { - "display_name": "atomate2", + "name": "python3", "language": "python", - "name": "atomate2" + "display_name": "Python 3 (ipykernel)" }, "language_info": { "codemirror_mode": { From 03e7e2ee88990d8b18c97c541fd769018eef41b0 Mon Sep 17 00:00:00 2001 From: JaGeo Date: Thu, 23 Nov 2023 23:55:35 +0100 Subject: [PATCH 19/26] cleanup --- docs/tutorials/5-dynamic-flows.ipynb | 250 +++++++++++++-------------- 1 file changed, 117 insertions(+), 133 deletions(-) diff --git a/docs/tutorials/5-dynamic-flows.ipynb b/docs/tutorials/5-dynamic-flows.ipynb index 9673cd21..43844076 100644 --- a/docs/tutorials/5-dynamic-flows.ipynb +++ b/docs/tutorials/5-dynamic-flows.ipynb @@ -39,14 +39,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "78348798", "metadata": { "nbsphinx": "hidden", "tags": [], "ExecuteTime": { - "end_time": "2023-11-23T22:46:57.984085360Z", - "start_time": "2023-11-23T22:46:57.166923399Z" + "end_time": "2023-11-23T22:55:01.113171569Z", + "start_time": "2023-11-23T22:55:01.112486877Z" } }, "outputs": [], @@ -58,12 +58,12 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "id": "964a8c44", "metadata": { "ExecuteTime": { - "end_time": "2023-11-23T22:46:58.036098609Z", - "start_time": "2023-11-23T22:46:57.208641695Z" + "end_time": "2023-11-23T22:55:03.025973341Z", + "start_time": "2023-11-23T22:55:01.974313960Z" } }, "outputs": [ @@ -71,20 +71,20 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-11-23 23:46:57,177 INFO Started executing jobs locally\n", - "2023-11-23 23:46:57,179 INFO Starting job - make_list (16fc2fd0-c9c7-420d-b0e1-3e6b5802c744)\n", - "2023-11-23 23:46:57,182 INFO Finished job - make_list (16fc2fd0-c9c7-420d-b0e1-3e6b5802c744)\n", - "2023-11-23 23:46:57,183 INFO Starting job - add_distributed (536e3c78-7d59-43ed-9f4f-72ce9b3793ef)\n", - "2023-11-23 23:46:57,191 INFO Finished job - add_distributed (536e3c78-7d59-43ed-9f4f-72ce9b3793ef)\n", - "2023-11-23 23:46:57,193 INFO Starting job - add (44007968-8394-442f-ba96-d52dc30a80ce)\n", - "2023-11-23 23:46:57,195 INFO Finished job - add (44007968-8394-442f-ba96-d52dc30a80ce)\n", - "2023-11-23 23:46:57,196 INFO Starting job - add (1af57ed7-3594-4180-b550-787c5f029d9f)\n", - "2023-11-23 23:46:57,198 INFO Finished job - add (1af57ed7-3594-4180-b550-787c5f029d9f)\n", - "2023-11-23 23:46:57,199 INFO Starting job - add (f7ea9158-40f8-4439-9d18-b6b6fb3e9289)\n", - "2023-11-23 23:46:57,201 INFO Finished job - add (f7ea9158-40f8-4439-9d18-b6b6fb3e9289)\n", - "2023-11-23 23:46:57,202 INFO Starting job - add (72c752d3-208a-45ce-bbb6-625e4debed73)\n", - "2023-11-23 23:46:57,204 INFO Finished job - add (72c752d3-208a-45ce-bbb6-625e4debed73)\n", - "2023-11-23 23:46:57,205 INFO Finished executing jobs locally\n" + "2023-11-23 23:55:02,807 INFO Started executing jobs locally\n", + "2023-11-23 23:55:02,937 INFO Starting job - make_list (d509897c-c318-4b2e-889e-a77331480a58)\n", + "2023-11-23 23:55:02,955 INFO Finished job - make_list (d509897c-c318-4b2e-889e-a77331480a58)\n", + "2023-11-23 23:55:02,956 INFO Starting job - add_distributed (7e07bfa4-a0cc-4535-b6c8-e3ec78fe96fb)\n", + "2023-11-23 23:55:02,963 INFO Finished job - add_distributed (7e07bfa4-a0cc-4535-b6c8-e3ec78fe96fb)\n", + "2023-11-23 23:55:03,005 INFO Starting job - add (62d7192e-d2af-4f28-a790-ac4987bd9f41)\n", + "2023-11-23 23:55:03,008 INFO Finished job - add (62d7192e-d2af-4f28-a790-ac4987bd9f41)\n", + "2023-11-23 23:55:03,009 INFO Starting job - add (d8a7d4aa-8255-417f-9cc8-bc9e16896d33)\n", + "2023-11-23 23:55:03,012 INFO Finished job - add (d8a7d4aa-8255-417f-9cc8-bc9e16896d33)\n", + "2023-11-23 23:55:03,012 INFO Starting job - add (15ce2067-8138-4692-baf8-8ceada256139)\n", + "2023-11-23 23:55:03,014 INFO Finished job - add (15ce2067-8138-4692-baf8-8ceada256139)\n", + "2023-11-23 23:55:03,015 INFO Starting job - add (43477f07-28fd-4944-9ae7-02f41282c8d3)\n", + "2023-11-23 23:55:03,017 INFO Finished job - add (43477f07-28fd-4944-9ae7-02f41282c8d3)\n", + "2023-11-23 23:55:03,018 INFO Finished executing jobs locally\n" ] }, { @@ -127,12 +127,12 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "id": "82e368e6", "metadata": { "ExecuteTime": { - "end_time": "2023-11-23T22:46:58.037599975Z", - "start_time": "2023-11-23T22:46:57.233955503Z" + "end_time": "2023-11-23T22:55:03.075177954Z", + "start_time": "2023-11-23T22:55:03.026408190Z" } }, "outputs": [ @@ -140,16 +140,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "16fc2fd0-c9c7-420d-b0e1-3e6b5802c744 -> {1: Response(output=[2, 2, 2, 2], detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "536e3c78-7d59-43ed-9f4f-72ce9b3793ef -> {1: Response(output=None, detour=None, addition=None, replace=Flow(name='Flow', uuid='1d51c5df-174a-41c8-b017-98494c8411f1')\n", - "1. Job(name='add', uuid='44007968-8394-442f-ba96-d52dc30a80ce')\n", - "2. Job(name='add', uuid='1af57ed7-3594-4180-b550-787c5f029d9f')\n", - "3. Job(name='add', uuid='f7ea9158-40f8-4439-9d18-b6b6fb3e9289')\n", - "4. Job(name='add', uuid='72c752d3-208a-45ce-bbb6-625e4debed73'), stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "44007968-8394-442f-ba96-d52dc30a80ce -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "1af57ed7-3594-4180-b550-787c5f029d9f -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "f7ea9158-40f8-4439-9d18-b6b6fb3e9289 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "72c752d3-208a-45ce-bbb6-625e4debed73 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" + "d509897c-c318-4b2e-889e-a77331480a58 -> {1: Response(output=[2, 2, 2, 2], detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "7e07bfa4-a0cc-4535-b6c8-e3ec78fe96fb -> {1: Response(output=None, detour=None, addition=None, replace=Flow(name='Flow', uuid='ab9475e6-781b-4560-a0cc-4c260f80762a')\n", + "1. Job(name='add', uuid='62d7192e-d2af-4f28-a790-ac4987bd9f41')\n", + "2. Job(name='add', uuid='d8a7d4aa-8255-417f-9cc8-bc9e16896d33')\n", + "3. Job(name='add', uuid='15ce2067-8138-4692-baf8-8ceada256139')\n", + "4. Job(name='add', uuid='43477f07-28fd-4944-9ae7-02f41282c8d3'), stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "62d7192e-d2af-4f28-a790-ac4987bd9f41 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "d8a7d4aa-8255-417f-9cc8-bc9e16896d33 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "15ce2067-8138-4692-baf8-8ceada256139 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "43477f07-28fd-4944-9ae7-02f41282c8d3 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" ] } ], @@ -189,12 +189,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "id": "81870e55", "metadata": { "ExecuteTime": { - "end_time": "2023-11-23T22:46:58.041273240Z", - "start_time": "2023-11-23T22:46:57.406089273Z" + "end_time": "2023-11-23T22:55:06.857848628Z", + "start_time": "2023-11-23T22:55:06.789602522Z" } }, "outputs": [ @@ -202,14 +202,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-11-23 23:46:57,406 INFO Started executing jobs locally\n", - "2023-11-23 23:46:57,408 INFO Starting job - add (34ba86a3-99cd-461e-bc17-511fdf6e08ec)\n", - "2023-11-23 23:46:57,411 INFO Finished job - add (34ba86a3-99cd-461e-bc17-511fdf6e08ec)\n", - "2023-11-23 23:46:57,412 INFO Starting job - add_with_logic (349afb69-d732-44b8-b770-2500f44511f9)\n", - "2023-11-23 23:46:57,421 INFO Finished job - add_with_logic (349afb69-d732-44b8-b770-2500f44511f9)\n", - "2023-11-23 23:46:57,422 INFO Starting job - add (64e623da-cf31-485e-9ed1-1f996a955868)\n", - "2023-11-23 23:46:57,424 INFO Finished job - add (64e623da-cf31-485e-9ed1-1f996a955868)\n", - "2023-11-23 23:46:57,425 INFO Finished executing jobs locally\n" + "2023-11-23 23:55:06,796 INFO Started executing jobs locally\n", + "2023-11-23 23:55:06,797 INFO Starting job - add (5630b6ba-1e9a-42ff-be51-ece43fe76643)\n", + "2023-11-23 23:55:06,800 INFO Finished job - add (5630b6ba-1e9a-42ff-be51-ece43fe76643)\n", + "2023-11-23 23:55:06,801 INFO Starting job - add_with_logic (008b1973-6421-418f-b47d-8f32054fedc8)\n", + "2023-11-23 23:55:06,807 INFO Finished job - add_with_logic (008b1973-6421-418f-b47d-8f32054fedc8)\n", + "2023-11-23 23:55:06,808 INFO Starting job - add (4e43d6b6-523a-4760-b1cf-3f68916e80fd)\n", + "2023-11-23 23:55:06,812 INFO Finished job - add (4e43d6b6-523a-4760-b1cf-3f68916e80fd)\n", + "2023-11-23 23:55:06,813 INFO Finished executing jobs locally\n" ] } ], @@ -232,12 +232,12 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "id": "fdc68eb5", "metadata": { "ExecuteTime": { - "end_time": "2023-11-23T22:46:58.046206634Z", - "start_time": "2023-11-23T22:46:57.448864584Z" + "end_time": "2023-11-23T22:55:07.823489602Z", + "start_time": "2023-11-23T22:55:07.816907763Z" } }, "outputs": [ @@ -245,10 +245,10 @@ "name": "stdout", "output_type": "stream", "text": [ - "34ba86a3-99cd-461e-bc17-511fdf6e08ec -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "349afb69-d732-44b8-b770-2500f44511f9 -> {1: Response(output=None, detour=None, addition=Flow(name='Flow', uuid='20581dd8-b00e-424b-94ad-7f2b2f4a240b')\n", - "1. Job(name='add', uuid='64e623da-cf31-485e-9ed1-1f996a955868'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "64e623da-cf31-485e-9ed1-1f996a955868 -> {1: Response(output=5, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" + "5630b6ba-1e9a-42ff-be51-ece43fe76643 -> {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "008b1973-6421-418f-b47d-8f32054fedc8 -> {1: Response(output=None, detour=None, addition=Flow(name='Flow', uuid='1994a56f-6dd8-4e35-a187-76d27d808d09')\n", + "1. Job(name='add', uuid='4e43d6b6-523a-4760-b1cf-3f68916e80fd'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "4e43d6b6-523a-4760-b1cf-3f68916e80fd -> {1: Response(output=5, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" ] } ], @@ -267,12 +267,12 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "id": "27569bd7", "metadata": { "ExecuteTime": { - "end_time": "2023-11-23T22:46:58.090608163Z", - "start_time": "2023-11-23T22:46:57.481314817Z" + "end_time": "2023-11-23T22:55:09.827238198Z", + "start_time": "2023-11-23T22:55:09.755367636Z" } }, "outputs": [ @@ -280,12 +280,12 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-11-23 23:46:57,479 INFO Started executing jobs locally\n", - "2023-11-23 23:46:57,481 INFO Starting job - add (0ae20220-b69c-4e6b-827a-2f046b00d504)\n", - "2023-11-23 23:46:57,484 INFO Finished job - add (0ae20220-b69c-4e6b-827a-2f046b00d504)\n", - "2023-11-23 23:46:57,485 INFO Starting job - add_with_logic (145dea09-be6c-47fc-ac46-c9a6908323e7)\n", - "2023-11-23 23:46:57,490 INFO Finished job - add_with_logic (145dea09-be6c-47fc-ac46-c9a6908323e7)\n", - "2023-11-23 23:46:57,491 INFO Finished executing jobs locally\n" + "2023-11-23 23:55:09,755 INFO Started executing jobs locally\n", + "2023-11-23 23:55:09,756 INFO Starting job - add (92e3f817-1350-4492-aeee-ec3501bce66f)\n", + "2023-11-23 23:55:09,759 INFO Finished job - add (92e3f817-1350-4492-aeee-ec3501bce66f)\n", + "2023-11-23 23:55:09,759 INFO Starting job - add_with_logic (e20fe185-4e34-4422-914f-e0acb4cbd529)\n", + "2023-11-23 23:55:09,764 INFO Finished job - add_with_logic (e20fe185-4e34-4422-914f-e0acb4cbd529)\n", + "2023-11-23 23:55:09,765 INFO Finished executing jobs locally\n" ] } ], @@ -308,12 +308,12 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "id": "b8d0c011", "metadata": { "ExecuteTime": { - "end_time": "2023-11-23T22:46:58.092445839Z", - "start_time": "2023-11-23T22:46:57.524568954Z" + "end_time": "2023-11-23T22:55:10.647133331Z", + "start_time": "2023-11-23T22:55:10.639661076Z" } }, "outputs": [ @@ -321,8 +321,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "0ae20220-b69c-4e6b-827a-2f046b00d504 -> {1: Response(output=21, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", - "145dea09-be6c-47fc-ac46-c9a6908323e7 -> {1: Response(output=None, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" + "92e3f817-1350-4492-aeee-ec3501bce66f -> {1: Response(output=21, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n", + "e20fe185-4e34-4422-914f-e0acb4cbd529 -> {1: Response(output=None, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}\n" ] } ], @@ -350,59 +350,44 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "2023-11-23 23:46:57,625 INFO Started executing jobs locally\n", - "2023-11-23 23:46:57,626 INFO Starting job - fibonacci (81d7944a-bd5a-4edd-a67f-36d2312e506b)\n", - "2023-11-23 23:46:57,630 INFO Finished job - fibonacci (81d7944a-bd5a-4edd-a67f-36d2312e506b)\n", - "2023-11-23 23:46:57,631 INFO Starting job - fibonacci (3219e168-b719-409a-8dbd-94370c03e162)\n", - "2023-11-23 23:46:57,634 INFO Finished job - fibonacci (3219e168-b719-409a-8dbd-94370c03e162)\n", - "2023-11-23 23:46:57,636 INFO Starting job - fibonacci (98308b42-817b-4ee5-bf68-fefcf1bd0de3)\n", - "2023-11-23 23:46:57,639 INFO Finished job - fibonacci (98308b42-817b-4ee5-bf68-fefcf1bd0de3)\n", - "2023-11-23 23:46:57,640 INFO Starting job - fibonacci (57c73c23-9fbf-4f08-8e0f-7698027b632b)\n", - "2023-11-23 23:46:57,643 INFO Finished job - fibonacci (57c73c23-9fbf-4f08-8e0f-7698027b632b)\n", - "2023-11-23 23:46:57,644 INFO Starting job - fibonacci (33c81f02-e87b-4848-8780-a09de20041b1)\n", - "2023-11-23 23:46:57,647 INFO Finished job - fibonacci (33c81f02-e87b-4848-8780-a09de20041b1)\n", - "2023-11-23 23:46:57,648 INFO Starting job - fibonacci (690e4d45-9e9b-43d8-8865-2085cc0feee2)\n", - "2023-11-23 23:46:57,650 INFO Finished job - fibonacci (690e4d45-9e9b-43d8-8865-2085cc0feee2)\n", - "2023-11-23 23:46:57,651 INFO Starting job - fibonacci (8142cc72-a4e3-4884-a446-98f0bea98569)\n", - "2023-11-23 23:46:57,655 INFO Finished job - fibonacci (8142cc72-a4e3-4884-a446-98f0bea98569)\n", - "2023-11-23 23:46:57,656 INFO Starting job - fibonacci (c2b4ec59-5088-4daf-9dba-34a2a3fbe56e)\n", - "2023-11-23 23:46:57,659 INFO Finished job - fibonacci (c2b4ec59-5088-4daf-9dba-34a2a3fbe56e)\n", - "2023-11-23 23:46:57,660 INFO Starting job - fibonacci (2db909a8-14df-4bd1-89a8-d188948382f1)\n", - "2023-11-23 23:46:57,663 INFO Finished job - fibonacci (2db909a8-14df-4bd1-89a8-d188948382f1)\n", - "2023-11-23 23:46:57,664 INFO Starting job - fibonacci (273a46cc-4762-495e-a1d7-e4e26203f122)\n", - "2023-11-23 23:46:57,666 INFO Finished job - fibonacci (273a46cc-4762-495e-a1d7-e4e26203f122)\n", - "2023-11-23 23:46:57,667 INFO Starting job - fibonacci (3fe6a08d-74f9-406f-a219-fdd68f1f6ec9)\n", - "2023-11-23 23:46:57,670 INFO Finished job - fibonacci (3fe6a08d-74f9-406f-a219-fdd68f1f6ec9)\n", - "2023-11-23 23:46:57,671 INFO Starting job - fibonacci (5c44497a-4497-4bf2-a0ee-172abf1251eb)\n", - "2023-11-23 23:46:57,674 INFO Finished job - fibonacci (5c44497a-4497-4bf2-a0ee-172abf1251eb)\n", - "2023-11-23 23:46:57,675 INFO Starting job - fibonacci (1b21a0be-8b79-42ad-a13b-c43c7077d637)\n", - "2023-11-23 23:46:57,678 INFO Finished job - fibonacci (1b21a0be-8b79-42ad-a13b-c43c7077d637)\n", - "2023-11-23 23:46:57,679 INFO Starting job - fibonacci (a6af5fcc-2a3b-4a40-8620-28edadb64b9f)\n", - "2023-11-23 23:46:57,682 INFO Finished job - fibonacci (a6af5fcc-2a3b-4a40-8620-28edadb64b9f)\n", - "2023-11-23 23:46:57,683 INFO Starting job - fibonacci (552ad89c-3e1f-4385-8eb5-01a761a6f9f1)\n", - "2023-11-23 23:46:57,686 INFO Finished job - fibonacci (552ad89c-3e1f-4385-8eb5-01a761a6f9f1)\n", - "2023-11-23 23:46:57,687 INFO Finished executing jobs locally\n", - "{'81d7944a-bd5a-4edd-a67f-36d2312e506b': {1: Response(output=2, detour=None, addition=Flow(name='Flow', uuid='b497505f-e73a-4324-99b4-2923d76bed6e')\n", - "1. Job(name='fibonacci', uuid='3219e168-b719-409a-8dbd-94370c03e162'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '3219e168-b719-409a-8dbd-94370c03e162': {1: Response(output=3, detour=None, addition=Flow(name='Flow', uuid='985aad5f-060f-4841-a2d0-edcdb4a4526d')\n", - "1. Job(name='fibonacci', uuid='98308b42-817b-4ee5-bf68-fefcf1bd0de3'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '98308b42-817b-4ee5-bf68-fefcf1bd0de3': {1: Response(output=5, detour=None, addition=Flow(name='Flow', uuid='076cc046-7ffe-40fe-a14d-5c97d9e92b40')\n", - "1. Job(name='fibonacci', uuid='57c73c23-9fbf-4f08-8e0f-7698027b632b'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '57c73c23-9fbf-4f08-8e0f-7698027b632b': {1: Response(output=8, detour=None, addition=Flow(name='Flow', uuid='38b57dd4-ddb1-425c-8153-5e2c195942da')\n", - "1. Job(name='fibonacci', uuid='33c81f02-e87b-4848-8780-a09de20041b1'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '33c81f02-e87b-4848-8780-a09de20041b1': {1: Response(output=13, detour=None, addition=Flow(name='Flow', uuid='057e0e16-62d3-461e-a13b-9d86c5f72780')\n", - "1. Job(name='fibonacci', uuid='690e4d45-9e9b-43d8-8865-2085cc0feee2'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '690e4d45-9e9b-43d8-8865-2085cc0feee2': {1: Response(output=21, detour=None, addition=Flow(name='Flow', uuid='a2f5bb27-5539-48be-a706-04af3756eedf')\n", - "1. Job(name='fibonacci', uuid='8142cc72-a4e3-4884-a446-98f0bea98569'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '8142cc72-a4e3-4884-a446-98f0bea98569': {1: Response(output=34, detour=None, addition=Flow(name='Flow', uuid='c0b854aa-aae5-4277-b7e1-45416e65384e')\n", - "1. Job(name='fibonacci', uuid='c2b4ec59-5088-4daf-9dba-34a2a3fbe56e'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, 'c2b4ec59-5088-4daf-9dba-34a2a3fbe56e': {1: Response(output=55, detour=None, addition=Flow(name='Flow', uuid='f31f638f-4147-4677-b8f5-72e44d9b12f9')\n", - "1. Job(name='fibonacci', uuid='2db909a8-14df-4bd1-89a8-d188948382f1'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '2db909a8-14df-4bd1-89a8-d188948382f1': {1: Response(output=89, detour=None, addition=Flow(name='Flow', uuid='5f5383a3-7b2d-4e86-98ca-66636006e6a3')\n", - "1. Job(name='fibonacci', uuid='273a46cc-4762-495e-a1d7-e4e26203f122'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '273a46cc-4762-495e-a1d7-e4e26203f122': {1: Response(output=144, detour=None, addition=Flow(name='Flow', uuid='4711a431-8976-4e7c-be1c-cb4962aab4a7')\n", - "1. Job(name='fibonacci', uuid='3fe6a08d-74f9-406f-a219-fdd68f1f6ec9'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '3fe6a08d-74f9-406f-a219-fdd68f1f6ec9': {1: Response(output=233, detour=None, addition=Flow(name='Flow', uuid='0dc7cb69-5797-4880-a8b7-c54f99490322')\n", - "1. Job(name='fibonacci', uuid='5c44497a-4497-4bf2-a0ee-172abf1251eb'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '5c44497a-4497-4bf2-a0ee-172abf1251eb': {1: Response(output=377, detour=None, addition=Flow(name='Flow', uuid='2469da06-bb02-4597-925c-df154d34115d')\n", - "1. Job(name='fibonacci', uuid='1b21a0be-8b79-42ad-a13b-c43c7077d637'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '1b21a0be-8b79-42ad-a13b-c43c7077d637': {1: Response(output=610, detour=None, addition=Flow(name='Flow', uuid='c92c422d-372c-45bd-9159-1ef9243c8f7f')\n", - "1. Job(name='fibonacci', uuid='a6af5fcc-2a3b-4a40-8620-28edadb64b9f'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, 'a6af5fcc-2a3b-4a40-8620-28edadb64b9f': {1: Response(output=987, detour=None, addition=Flow(name='Flow', uuid='4e7d136c-a125-4c08-b660-247bea293d64')\n", - "1. Job(name='fibonacci', uuid='552ad89c-3e1f-4385-8eb5-01a761a6f9f1'), replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}, '552ad89c-3e1f-4385-8eb5-01a761a6f9f1': {1: Response(output=1597, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}}\n" + "2023-11-23 23:55:13,324 INFO Started executing jobs locally\n", + "2023-11-23 23:55:13,325 INFO Starting job - fibonacci (6f7a1e92-577f-4289-85e5-e07a82e542db)\n", + "2023-11-23 23:55:13,329 INFO Finished job - fibonacci (6f7a1e92-577f-4289-85e5-e07a82e542db)\n", + "2023-11-23 23:55:13,329 INFO Starting job - fibonacci (862ce31a-a963-419a-9862-f419c47c3c9e)\n", + "2023-11-23 23:55:13,332 INFO Finished job - fibonacci (862ce31a-a963-419a-9862-f419c47c3c9e)\n", + "2023-11-23 23:55:13,333 INFO Starting job - fibonacci (81c28a0a-f1ad-487c-b484-46729f6ca9a6)\n", + "2023-11-23 23:55:13,335 INFO Finished job - fibonacci (81c28a0a-f1ad-487c-b484-46729f6ca9a6)\n", + "2023-11-23 23:55:13,336 INFO Starting job - fibonacci (084e2022-5a00-4ca8-9edf-46977ca08388)\n", + "2023-11-23 23:55:13,338 INFO Finished job - fibonacci (084e2022-5a00-4ca8-9edf-46977ca08388)\n", + "2023-11-23 23:55:13,339 INFO Starting job - fibonacci (382f371c-9e92-4185-8dbb-829801ec1afd)\n", + "2023-11-23 23:55:13,342 INFO Finished job - fibonacci (382f371c-9e92-4185-8dbb-829801ec1afd)\n", + "2023-11-23 23:55:13,342 INFO Starting job - fibonacci (f4521225-993a-4b42-9818-8aaff0da9181)\n", + "2023-11-23 23:55:13,345 INFO Finished job - fibonacci (f4521225-993a-4b42-9818-8aaff0da9181)\n", + "2023-11-23 23:55:13,346 INFO Starting job - fibonacci (3d475e3e-3774-4015-b056-a23f5cf1da6a)\n", + "2023-11-23 23:55:13,349 INFO Finished job - fibonacci (3d475e3e-3774-4015-b056-a23f5cf1da6a)\n", + "2023-11-23 23:55:13,351 INFO Starting job - fibonacci (ddc61891-1ce1-4d96-b66e-cb60d52e642a)\n", + "2023-11-23 23:55:13,353 INFO Finished job - fibonacci (ddc61891-1ce1-4d96-b66e-cb60d52e642a)\n", + "2023-11-23 23:55:13,354 INFO Starting job - fibonacci (499ab638-d8cc-4272-b2b4-f64d246f3676)\n", + "2023-11-23 23:55:13,357 INFO Finished job - fibonacci (499ab638-d8cc-4272-b2b4-f64d246f3676)\n", + "2023-11-23 23:55:13,357 INFO Starting job - fibonacci (71b7d00f-0109-4a23-8a54-fb29ef143cd2)\n", + "2023-11-23 23:55:13,360 INFO Finished job - fibonacci (71b7d00f-0109-4a23-8a54-fb29ef143cd2)\n", + "2023-11-23 23:55:13,361 INFO Starting job - fibonacci (d60f3f5d-821a-434a-a8c1-50468b381d5f)\n", + "2023-11-23 23:55:13,363 INFO Finished job - fibonacci (d60f3f5d-821a-434a-a8c1-50468b381d5f)\n", + "2023-11-23 23:55:13,364 INFO Starting job - fibonacci (7828008e-2f89-4226-aaac-66f25d4ccf0a)\n", + "2023-11-23 23:55:13,367 INFO Finished job - fibonacci (7828008e-2f89-4226-aaac-66f25d4ccf0a)\n", + "2023-11-23 23:55:13,367 INFO Starting job - fibonacci (062b0cad-2b62-4f42-bd39-ac1b3c90d141)\n", + "2023-11-23 23:55:13,370 INFO Finished job - fibonacci (062b0cad-2b62-4f42-bd39-ac1b3c90d141)\n", + "2023-11-23 23:55:13,371 INFO Starting job - fibonacci (6c1bf8e4-1808-42b4-a99e-49dd031894bd)\n", + "2023-11-23 23:55:13,373 INFO Finished job - fibonacci (6c1bf8e4-1808-42b4-a99e-49dd031894bd)\n", + "2023-11-23 23:55:13,374 INFO Starting job - fibonacci (d82e352d-1ca3-40d2-85d1-3f08989d3551)\n", + "2023-11-23 23:55:13,376 INFO Finished job - fibonacci (d82e352d-1ca3-40d2-85d1-3f08989d3551)\n", + "2023-11-23 23:55:13,377 INFO Finished executing jobs locally\n" ] } ], @@ -430,14 +415,13 @@ "fibonacci_job = fibonacci(1, 1)\n", "\n", "# run the job; responses will contain the output from all jobs\n", - "responses = run_locally(fibonacci_job)\n", - "print(responses)\n" + "responses = run_locally(fibonacci_job)\n" ], "metadata": { "collapsed": false, "ExecuteTime": { - "end_time": "2023-11-23T22:46:58.136876182Z", - "start_time": "2023-11-23T22:46:57.625858812Z" + "end_time": "2023-11-23T22:55:13.426518952Z", + "start_time": "2023-11-23T22:55:13.322421257Z" } } }, @@ -459,12 +443,12 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "id": "fee35a97", "metadata": { "ExecuteTime": { - "end_time": "2023-11-23T22:46:58.142624609Z", - "start_time": "2023-11-23T22:46:57.774129081Z" + "end_time": "2023-11-23T22:55:16.347154656Z", + "start_time": "2023-11-23T22:55:16.266053982Z" } }, "outputs": [ @@ -472,14 +456,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "2023-11-23 23:46:57,774 INFO Started executing jobs locally\n", - "2023-11-23 23:46:57,775 INFO Starting job - add (55465994-efcd-4835-b92e-f7ea36765268)\n", - "2023-11-23 23:46:57,778 INFO Finished job - add (55465994-efcd-4835-b92e-f7ea36765268)\n", - "2023-11-23 23:46:57,779 INFO Starting job - add_with_logic (d6cf8229-dbcc-422c-8504-177c433f48ac)\n", - "2023-11-23 23:46:57,786 INFO Finished job - add_with_logic (d6cf8229-dbcc-422c-8504-177c433f48ac)\n", - "2023-11-23 23:46:57,788 INFO Starting job - add (996e6af8-c56f-4779-8135-00493dbf9298)\n", - "2023-11-23 23:46:57,790 INFO Finished job - add (996e6af8-c56f-4779-8135-00493dbf9298)\n", - "2023-11-23 23:46:57,791 INFO Finished executing jobs locally\n" + "2023-11-23 23:55:16,275 INFO Started executing jobs locally\n", + "2023-11-23 23:55:16,278 INFO Starting job - add (301d75f0-7042-494a-9f24-cab0428c2fd1)\n", + "2023-11-23 23:55:16,281 INFO Finished job - add (301d75f0-7042-494a-9f24-cab0428c2fd1)\n", + "2023-11-23 23:55:16,282 INFO Starting job - add_with_logic (97be61a8-eec4-4e64-bf53-ba37621575e7)\n", + "2023-11-23 23:55:16,292 INFO Finished job - add_with_logic (97be61a8-eec4-4e64-bf53-ba37621575e7)\n", + "2023-11-23 23:55:16,293 INFO Starting job - add (d4c31f68-09ad-418a-ac52-89b303fc2a00)\n", + "2023-11-23 23:55:16,296 INFO Finished job - add (d4c31f68-09ad-418a-ac52-89b303fc2a00)\n", + "2023-11-23 23:55:16,296 INFO Finished executing jobs locally\n" ] } ], @@ -502,20 +486,20 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "id": "b3b7e8ba", "metadata": { "ExecuteTime": { - "end_time": "2023-11-23T22:46:58.143531724Z", - "start_time": "2023-11-23T22:46:57.816557390Z" + "end_time": "2023-11-23T22:55:17.024836601Z", + "start_time": "2023-11-23T22:55:17.019427206Z" } }, "outputs": [ { "data": { - "text/plain": "{'55465994-efcd-4835-b92e-f7ea36765268': {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)},\n 'd6cf8229-dbcc-422c-8504-177c433f48ac': {1: Response(output=None, detour=Flow(name='Flow', uuid='98311314-8087-4787-b2ea-c89478ff2999')\n 1. Job(name='add', uuid='996e6af8-c56f-4779-8135-00493dbf9298'), addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)},\n '996e6af8-c56f-4779-8135-00493dbf9298': {1: Response(output=5, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}}" + "text/plain": "{'301d75f0-7042-494a-9f24-cab0428c2fd1': {1: Response(output=3, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)},\n '97be61a8-eec4-4e64-bf53-ba37621575e7': {1: Response(output=None, detour=Flow(name='Flow', uuid='0de995a5-1110-4200-b010-276cb2017474')\n 1. Job(name='add', uuid='d4c31f68-09ad-418a-ac52-89b303fc2a00'), addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)},\n 'd4c31f68-09ad-418a-ac52-89b303fc2a00': {1: Response(output=5, detour=None, addition=None, replace=None, stored_data=None, stop_children=False, stop_jobflow=False)}}" }, - "execution_count": 11, + "execution_count": 10, "metadata": {}, "output_type": "execute_result" } From 3ee077eda8299f23cfceed8dd7f73c5989e6adf7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:27:05 +0000 Subject: [PATCH 20/26] Bump pydantic from 2.5.1 to 2.5.2 (#495) Bumps [pydantic](https://github.com/pydantic/pydantic) from 2.5.1 to 2.5.2. - [Release notes](https://github.com/pydantic/pydantic/releases) - [Changelog](https://github.com/pydantic/pydantic/blob/v2.5.2/HISTORY.md) - [Commits](https://github.com/pydantic/pydantic/compare/v2.5.1...v2.5.2) --- updated-dependencies: - dependency-name: pydantic dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d6cf8cc7..815b08b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ strict = [ "moto==4.2.9", "networkx==3.2.1", "pydantic-settings==2.0.3", - "pydantic==2.5.1", + "pydantic==2.5.2", "pydash==7.0.6", "pydot==1.4.2", "typing-extensions==4.8.0", From 1728c130d277c7951a9013265b79980d43e1337d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:27:09 +0000 Subject: [PATCH 21/26] Bump ipython from 8.17.2 to 8.18.1 (#496) Bumps [ipython](https://github.com/ipython/ipython) from 8.17.2 to 8.18.1. - [Release notes](https://github.com/ipython/ipython/releases) - [Commits](https://github.com/ipython/ipython/compare/8.17.2...8.18.1) --- updated-dependencies: - dependency-name: ipython dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 815b08b8..c0074f79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,7 @@ dependencies = [ docs = [ "autodoc_pydantic==2.0.1", "furo==2023.9.10", - "ipython==8.17.2", + "ipython==8.18.1", "myst_parser==2.0.0", "nbsphinx==0.9.3", "sphinx-copybutton==0.5.2", From ce2a7a63f141a71c0ae6dc5247ed8906c85f81c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:27:26 +0000 Subject: [PATCH 22/26] Bump moto from 4.2.9 to 4.2.10 (#497) Bumps [moto](https://github.com/getmoto/moto) from 4.2.9 to 4.2.10. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/4.2.9...4.2.10) --- updated-dependencies: - dependency-name: moto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c0074f79..fb361e9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ docs = [ "sphinx==7.2.6", ] dev = ["pre-commit>=2.12.1"] -tests = ["moto==4.2.9", "pytest-cov==4.1.0", "pytest==7.4.3"] +tests = ["moto==4.2.10", "pytest-cov==4.1.0", "pytest==7.4.3"] vis = ["matplotlib", "pydot"] fireworks = ["FireWorks"] strict = [ @@ -55,7 +55,7 @@ strict = [ "maggma==0.57.10", "matplotlib==3.8.2", "monty==2023.11.3", - "moto==4.2.9", + "moto==4.2.10", "networkx==3.2.1", "pydantic-settings==2.0.3", "pydantic==2.5.2", From 3b180882a2c2d70a85c6ae434f7c4f44bdfdca7e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:27:37 +0000 Subject: [PATCH 23/26] Bump maggma from 0.57.10 to 0.58.0 (#498) Bumps [maggma](https://github.com/materialsproject/maggma) from 0.57.10 to 0.58.0. - [Release notes](https://github.com/materialsproject/maggma/releases) - [Changelog](https://github.com/materialsproject/maggma/blob/main/docs/CHANGELOG.md) - [Commits](https://github.com/materialsproject/maggma/compare/v0.57.10...v0.58.0) --- updated-dependencies: - dependency-name: maggma dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index fb361e9b..82895c7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,7 +52,7 @@ fireworks = ["FireWorks"] strict = [ "FireWorks==2.0.3", "PyYAML==6.0.1", - "maggma==0.57.10", + "maggma==0.58.0", "matplotlib==3.8.2", "monty==2023.11.3", "moto==4.2.10", From 6439853ba3a393b33e0651f98ca3520f88690dcb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Nov 2023 23:29:30 +0000 Subject: [PATCH 24/26] Bump pydantic-settings from 2.0.3 to 2.1.0 (#492) Bumps [pydantic-settings](https://github.com/pydantic/pydantic-settings) from 2.0.3 to 2.1.0. - [Release notes](https://github.com/pydantic/pydantic-settings/releases) - [Commits](https://github.com/pydantic/pydantic-settings/compare/v2.0.3...v2.1.0) --- updated-dependencies: - dependency-name: pydantic-settings dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 82895c7a..7ca63aaa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,7 @@ strict = [ "monty==2023.11.3", "moto==4.2.10", "networkx==3.2.1", - "pydantic-settings==2.0.3", + "pydantic-settings==2.1.0", "pydantic==2.5.2", "pydash==7.0.6", "pydot==1.4.2", From 46e10db66c6fbbf3e432138f64e76203a8c786b6 Mon Sep 17 00:00:00 2001 From: Guido Petretto Date: Tue, 28 Nov 2023 16:37:04 +0100 Subject: [PATCH 25/26] fix Flow deserialization for hosts --- src/jobflow/core/flow.py | 3 ++- tests/core/test_flow.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/jobflow/core/flow.py b/src/jobflow/core/flow.py index 8e865a12..7f9e911a 100644 --- a/src/jobflow/core/flow.py +++ b/src/jobflow/core/flow.py @@ -794,7 +794,8 @@ def add_jobs(self, jobs: Job | Flow | Sequence[Flow | Job]) -> None: f"current Flow ({self.uuid})" ) job_ids.add(job.uuid) - job.add_hosts_uuids(hosts) + if job.host != self.uuid: + job.add_hosts_uuids(hosts) self._jobs += tuple(jobs) def remove_jobs(self, indices: int | list[int]): diff --git a/tests/core/test_flow.py b/tests/core/test_flow.py index 54295b12..6f0d78f1 100644 --- a/tests/core/test_flow.py +++ b/tests/core/test_flow.py @@ -497,6 +497,7 @@ def test_serialization(): decoded_flow = MontyDecoder().process_decoded(encoded_flow) assert decoded_flow[0].host == host_uuid + assert flow_host.jobs[0].hosts == decoded_flow.jobs[0].hosts def test_update_kwargs(): From 003ec3a0b2c0550c63f8fd5bc91a4e5c6ae191ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 23:47:29 +0000 Subject: [PATCH 26/26] Bump moto from 4.2.10 to 4.2.11 (#500) Bumps [moto](https://github.com/getmoto/moto) from 4.2.10 to 4.2.11. - [Release notes](https://github.com/getmoto/moto/releases) - [Changelog](https://github.com/getmoto/moto/blob/master/CHANGELOG.md) - [Commits](https://github.com/getmoto/moto/compare/4.2.10...4.2.11) --- updated-dependencies: - dependency-name: moto dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7ca63aaa..3ac99790 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ docs = [ "sphinx==7.2.6", ] dev = ["pre-commit>=2.12.1"] -tests = ["moto==4.2.10", "pytest-cov==4.1.0", "pytest==7.4.3"] +tests = ["moto==4.2.11", "pytest-cov==4.1.0", "pytest==7.4.3"] vis = ["matplotlib", "pydot"] fireworks = ["FireWorks"] strict = [ @@ -55,7 +55,7 @@ strict = [ "maggma==0.58.0", "matplotlib==3.8.2", "monty==2023.11.3", - "moto==4.2.10", + "moto==4.2.11", "networkx==3.2.1", "pydantic-settings==2.1.0", "pydantic==2.5.2",