From a548187f883d9d75f7978daceabfd71e8873ec53 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Thu, 19 Dec 2024 18:10:24 +0900 Subject: [PATCH 1/7] docker: fix setting tags --- .actions/assistant.py | 19 +++++++++++++++++++ .github/workflows/docker-build.yml | 26 ++++++++------------------ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 6bb0bc201ed05..9c105cb38bf19 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -481,6 +481,25 @@ def convert_version2nightly(ver_file: str = "src/version.info") -> None: with open(ver_file, "w") as fo: fo.write(version + os.linesep) + @staticmethod + def generate_docker_tags( + release_version: str, + python_version: str, + torch_version: str, + cuda_version: str, + docker_project: str = "pytorchlightning/pytorch_lightning", + add_latest: bool = False, + ) -> str: + """Generate docker tags for the given versions.""" + tags = [f"latest-py{python_version}-torch{torch_version}-cuda{cuda_version}"] + if release_version: + tags += [f"{release_version}-py{python_version}-torch{torch_version}-cuda{cuda_version}"] + if add_latest: + tags += ["latest"] + + tags = [f"{docker_project}:{tag}" for tag in tags] + return ",".join(tags) + if __name__ == "__main__": import sys diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 09ae3adc45ac6..c8080719ef214 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -47,7 +47,7 @@ jobs: - { python_version: "3.11", pytorch_version: "2.2", cuda_version: "12.1.0" } - { python_version: "3.11", pytorch_version: "2.3", cuda_version: "12.1.0" } - { python_version: "3.11", pytorch_version: "2.4", cuda_version: "12.1.0" } - - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.0" } + - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.0" , latest: true} steps: - uses: actions/checkout@v4 with: @@ -65,23 +65,13 @@ jobs: run: echo "RELEASE_VERSION=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - name: Set tags run: | - import os - - repo = "pytorchlightning/pytorch_lightning" - ver = os.getenv('RELEASE_VERSION') - py_ver = "${{ matrix.python_version }}" - pt_ver = "${{ matrix.pytorch_version }}" - cuda_ver = "${{ matrix.cuda_version }}" - tags = [f"latest-py{py_ver}-torch{pt_ver}-cuda{cuda_ver}"] - if ver: - tags += [f"{ver}-py{py_ver}-torch{pt_ver}-cuda{cuda_ver}"] - if py_ver == '3.11' and pt_ver == '2.3' and cuda_ver == '12.1.0': - tags += ["latest"] - - tags = [f"{repo}:{tag}" for tag in tags] - with open(os.getenv('GITHUB_ENV'), "a") as gh_env: - gh_env.write("DOCKER_TAGS=" + ",".join(tags)) - shell: python + tags = $(python .actions/assistant.py generate_docker_tags \ + --release-version="${{ env.RELEASE_VERSION }}" \ + --python-version="${{ matrix.python_version }}" \ + --torch-version="${{ matrix.pytorch_version }}" \ + --cuda-version="${{ matrix.cuda_version }}" \ + --add_latest="${{ matrix.latest }}") + echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV - uses: docker/build-push-action@v6 with: From ef9661ba5bf2b49f6a026d615ec856577eca5156 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 19 Dec 2024 09:12:08 +0000 Subject: [PATCH 2/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index c8080719ef214..8141ac6f308e3 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -47,7 +47,7 @@ jobs: - { python_version: "3.11", pytorch_version: "2.2", cuda_version: "12.1.0" } - { python_version: "3.11", pytorch_version: "2.3", cuda_version: "12.1.0" } - { python_version: "3.11", pytorch_version: "2.4", cuda_version: "12.1.0" } - - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.0" , latest: true} + - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.0", latest: true } steps: - uses: actions/checkout@v4 with: From e59763ea78dd9f58762fcd6318d9b19ff01c695e Mon Sep 17 00:00:00 2001 From: Jirka B Date: Thu, 19 Dec 2024 18:12:32 +0900 Subject: [PATCH 3/7] pip install --- .github/workflows/docker-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index c8080719ef214..105b36a8f826f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -65,6 +65,7 @@ jobs: run: echo "RELEASE_VERSION=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV - name: Set tags run: | + pip install -q -r .actions/requirements.txt tags = $(python .actions/assistant.py generate_docker_tags \ --release-version="${{ env.RELEASE_VERSION }}" \ --python-version="${{ matrix.python_version }}" \ From de8a077a80bba2a7140516a65cfbac74767f34b6 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Thu, 19 Dec 2024 18:36:30 +0900 Subject: [PATCH 4/7] cmd --- .github/workflows/docker-build.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 477866a06fa60..673dd70045f7c 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -47,7 +47,7 @@ jobs: - { python_version: "3.11", pytorch_version: "2.2", cuda_version: "12.1.0" } - { python_version: "3.11", pytorch_version: "2.3", cuda_version: "12.1.0" } - { python_version: "3.11", pytorch_version: "2.4", cuda_version: "12.1.0" } - - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.0", latest: true } + - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.0", tag: "--add_latest" } steps: - uses: actions/checkout@v4 with: @@ -67,11 +67,10 @@ jobs: run: | pip install -q -r .actions/requirements.txt tags = $(python .actions/assistant.py generate_docker_tags \ - --release-version="${{ env.RELEASE_VERSION }}" \ - --python-version="${{ matrix.python_version }}" \ - --torch-version="${{ matrix.pytorch_version }}" \ - --cuda-version="${{ matrix.cuda_version }}" \ - --add_latest="${{ matrix.latest }}") + --release_version="${{ env.RELEASE_VERSION }}" \ + --python_version="${{ matrix.python_version }}" \ + --torch_version="${{ matrix.pytorch_version }}" \ + --cuda_version="${{ matrix.cuda_version }}" ${{ matrix.tag }} ) echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV - uses: docker/build-push-action@v6 From 462197a0a184b771abcfd87684149f951086b43b Mon Sep 17 00:00:00 2001 From: Jirka B Date: Thu, 19 Dec 2024 18:40:30 +0900 Subject: [PATCH 5/7] latest --- .github/workflows/docker-build.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 673dd70045f7c..89ee6495bf854 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -47,7 +47,7 @@ jobs: - { python_version: "3.11", pytorch_version: "2.2", cuda_version: "12.1.0" } - { python_version: "3.11", pytorch_version: "2.3", cuda_version: "12.1.0" } - { python_version: "3.11", pytorch_version: "2.4", cuda_version: "12.1.0" } - - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.0", tag: "--add_latest" } + - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.0", latest: "true" } steps: - uses: actions/checkout@v4 with: @@ -70,7 +70,8 @@ jobs: --release_version="${{ env.RELEASE_VERSION }}" \ --python_version="${{ matrix.python_version }}" \ --torch_version="${{ matrix.pytorch_version }}" \ - --cuda_version="${{ matrix.cuda_version }}" ${{ matrix.tag }} ) + --cuda_version="${{ matrix.cuda_version }}" \ + --add_latest="${{ matrix.latest || 'false' }}") echo "DOCKER_TAGS=$tags" >> $GITHUB_ENV - uses: docker/build-push-action@v6 From a58d1418ae92b778dacc349181ab6155298d7291 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Thu, 19 Dec 2024 18:43:53 +0900 Subject: [PATCH 6/7] print --- .actions/assistant.py | 4 ++-- .github/workflows/docker-build.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.actions/assistant.py b/.actions/assistant.py index 9c105cb38bf19..41002ee5f5e45 100644 --- a/.actions/assistant.py +++ b/.actions/assistant.py @@ -489,7 +489,7 @@ def generate_docker_tags( cuda_version: str, docker_project: str = "pytorchlightning/pytorch_lightning", add_latest: bool = False, - ) -> str: + ) -> None: """Generate docker tags for the given versions.""" tags = [f"latest-py{python_version}-torch{torch_version}-cuda{cuda_version}"] if release_version: @@ -498,7 +498,7 @@ def generate_docker_tags( tags += ["latest"] tags = [f"{docker_project}:{tag}" for tag in tags] - return ",".join(tags) + print(",".join(tags)) if __name__ == "__main__": diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 89ee6495bf854..93f470f5e9d0a 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -66,7 +66,7 @@ jobs: - name: Set tags run: | pip install -q -r .actions/requirements.txt - tags = $(python .actions/assistant.py generate_docker_tags \ + tags=$(python .actions/assistant.py generate_docker_tags \ --release_version="${{ env.RELEASE_VERSION }}" \ --python_version="${{ matrix.python_version }}" \ --torch_version="${{ matrix.pytorch_version }}" \ From cee72ff52c289a3e5c01e729be46a16daad4ab30 Mon Sep 17 00:00:00 2001 From: Jirka B Date: Fri, 20 Dec 2024 15:00:37 +0900 Subject: [PATCH 7/7] cuda 12.1.1 --- .github/checkgroup.yml | 20 ++++++++++---------- .github/workflows/docker-build.yml | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.github/checkgroup.yml b/.github/checkgroup.yml index 6c7c6aa5f7c1b..b1d54bc5e12fc 100644 --- a/.github/checkgroup.yml +++ b/.github/checkgroup.yml @@ -145,17 +145,17 @@ subprojects: - "!*.md" - "!**/*.md" checks: - - "build-cuda (3.10, 2.1.2, 12.1.0)" - - "build-cuda (3.11, 2.2.2, 12.1.0)" - - "build-cuda (3.11, 2.3.1, 12.1.0)" - - "build-cuda (3.11, 2.4.1, 12.1.0)" - - "build-cuda (3.12, 2.5.1, 12.1.0)" + - "build-cuda (3.10, 2.1.2, 12.1.1)" + - "build-cuda (3.11, 2.2.2, 12.1.1)" + - "build-cuda (3.11, 2.3.1, 12.1.1)" + - "build-cuda (3.11, 2.4.1, 12.1.1)" + - "build-cuda (3.12, 2.5.1, 12.1.1)" #- "build-NGC" - - "build-pl (3.10, 2.1, 12.1.0)" - - "build-pl (3.11, 2.2, 12.1.0)" - - "build-pl (3.11, 2.3, 12.1.0)" - - "build-pl (3.11, 2.4, 12.1.0)" - - "build-pl (3.12, 2.5, 12.1.0)" + - "build-pl (3.10, 2.1, 12.1.1)" + - "build-pl (3.11, 2.2, 12.1.1)" + - "build-pl (3.11, 2.3, 12.1.1)" + - "build-pl (3.11, 2.4, 12.1.1)" + - "build-pl (3.12, 2.5, 12.1.1, true)" # SECTION: lightning_fabric diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 93f470f5e9d0a..7ab558aa7b07f 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -43,11 +43,11 @@ jobs: include: # We only release one docker image per PyTorch version. # Make sure the matrix here matches the one below. - - { python_version: "3.10", pytorch_version: "2.1", cuda_version: "12.1.0" } - - { python_version: "3.11", pytorch_version: "2.2", cuda_version: "12.1.0" } - - { python_version: "3.11", pytorch_version: "2.3", cuda_version: "12.1.0" } - - { python_version: "3.11", pytorch_version: "2.4", cuda_version: "12.1.0" } - - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.0", latest: "true" } + - { python_version: "3.10", pytorch_version: "2.1", cuda_version: "12.1.1" } + - { python_version: "3.11", pytorch_version: "2.2", cuda_version: "12.1.1" } + - { python_version: "3.11", pytorch_version: "2.3", cuda_version: "12.1.1" } + - { python_version: "3.11", pytorch_version: "2.4", cuda_version: "12.1.1" } + - { python_version: "3.12", pytorch_version: "2.5", cuda_version: "12.1.1", latest: "true" } steps: - uses: actions/checkout@v4 with: @@ -95,11 +95,11 @@ jobs: include: # These are the base images for PL release docker images. # Make sure the matrix here matches the one above. - - { python_version: "3.10", pytorch_version: "2.1.2", cuda_version: "12.1.0" } - - { python_version: "3.11", pytorch_version: "2.2.2", cuda_version: "12.1.0" } - - { python_version: "3.11", pytorch_version: "2.3.1", cuda_version: "12.1.0" } - - { python_version: "3.11", pytorch_version: "2.4.1", cuda_version: "12.1.0" } - - { python_version: "3.12", pytorch_version: "2.5.1", cuda_version: "12.1.0" } + - { python_version: "3.10", pytorch_version: "2.1.2", cuda_version: "12.1.1" } + - { python_version: "3.11", pytorch_version: "2.2.2", cuda_version: "12.1.1" } + - { python_version: "3.11", pytorch_version: "2.3.1", cuda_version: "12.1.1" } + - { python_version: "3.11", pytorch_version: "2.4.1", cuda_version: "12.1.1" } + - { python_version: "3.12", pytorch_version: "2.5.1", cuda_version: "12.1.1" } steps: - uses: actions/checkout@v4 - uses: docker/setup-buildx-action@v3