From 2f09f267364a17048983775ccd185a5e85f3a029 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 28 Mar 2024 02:53:50 +0530 Subject: [PATCH 01/31] updated support for benchmark language versions Signed-off-by: Abhishek Kumar --- config/systems.json | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/config/systems.json b/config/systems.json index 8272078f..6e437638 100644 --- a/config/systems.json +++ b/config/systems.json @@ -17,8 +17,8 @@ "languages": { "python": { "base_images": { - "3.7": "python:3.7-slim", - "3.8": "python:3.8-slim" + "3.8": "python:3.8-slim", + "3.9": "python:3.9-slim" }, "images": [ "run", @@ -34,8 +34,8 @@ }, "nodejs": { "base_images": { - "12": "node:12-slim", - "14": "node:14-slim" + "18": "node:18-slim", + "20": "node:20-slim" }, "images": [ "run", @@ -56,11 +56,9 @@ "python": { "base_images": { "3.9": "amazon/aws-lambda-python:3.9", - "3.8": "amazon/aws-lambda-python:3.8", - "3.7": "amazon/aws-lambda-python:3.7" + "3.8": "amazon/aws-lambda-python:3.8" }, "versions": [ - "3.7", "3.8", "3.9" ], @@ -122,7 +120,9 @@ }, "nodejs": { "base_images": { - "14": "mcr.microsoft.com/azure-functions/node:3.0-node14" + "16": "mcr.microsoft.com/azure-functions/node:3.0-node16", + "18": "mcr.microsoft.com/azure-functions/node:3.0-node18", + "20": "mcr.microsoft.com/azure-functions/node:3.0-node20" }, "images": [ "build" @@ -150,9 +150,8 @@ "languages": { "python": { "base_images": { - "3.7": "ubuntu:22.04", - "3.8": "ubuntu:22.04", - "3.9": "ubuntu:22.04" + "3.9": "ubuntu:22.04", + "3.10": "ubuntu:22.04" }, "images": [ "build" @@ -170,10 +169,8 @@ }, "nodejs": { "base_images": { - "10": "gcr.io/google-appengine/nodejs", - "12": "gcr.io/google-appengine/nodejs", - "14": "gcr.io/google-appengine/nodejs", - "16": "gcr.io/google-appengine/nodejs" + "18": "gcr.io/google-appengine/nodejs", + "20": "gcr.io/google-appengine/nodejs" }, "images": [ "build" @@ -196,8 +193,9 @@ "languages": { "python": { "base_images": { - "3.7": "openwhisk/action-python-v3.7", - "3.9": "openwhisk/action-python-v3.9" + "3.9": "openwhisk/action-python-v3.9", + "3.10": "openwhisk/action-python-v3.10", + "3.11": "openwhisk/action-python-v3.11" }, "images": [ "function" @@ -216,8 +214,8 @@ }, "nodejs": { "base_images": { - "10": "openwhisk/action-nodejs-v10", - "12": "openwhisk/action-nodejs-v12" + "18": "openwhisk/action-nodejs-v18", + "20": "openwhisk/action-nodejs-v20" }, "images": [ "function" From 483fd88f12574b3821355858d0530cb006d1cafc Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 17 Apr 2024 09:30:43 +0530 Subject: [PATCH 02/31] added aws lambda python3.10 support Signed-off-by: Abhishek Kumar --- config/systems.json | 1 + 1 file changed, 1 insertion(+) diff --git a/config/systems.json b/config/systems.json index 6e437638..0c270597 100644 --- a/config/systems.json +++ b/config/systems.json @@ -55,6 +55,7 @@ "languages": { "python": { "base_images": { + "3.10": "amazon/aws-lambda-python:3.10", "3.9": "amazon/aws-lambda-python:3.9", "3.8": "amazon/aws-lambda-python:3.8" }, From a4ea1a8e47361a437c245491b5e0bb6b9397b727 Mon Sep 17 00:00:00 2001 From: prajinkhadka <34759925+prajinkhadka@users.noreply.github.com> Date: Wed, 29 May 2024 23:22:18 +0545 Subject: [PATCH 03/31] Create requirements.txt.3.10 for Python3.10 for Benchamark - 210.thumbnailer --- .../200.multimedia/210.thumbnailer/python/requirements.txt.3.10 | 1 + 1 file changed, 1 insertion(+) create mode 100644 benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.10 diff --git a/benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.10 b/benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.10 new file mode 100644 index 00000000..9caa46c8 --- /dev/null +++ b/benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.10 @@ -0,0 +1 @@ +pillow==10.3.0 From 877be279e826ecb013c74b83e2f36ef46bec291c Mon Sep 17 00:00:00 2001 From: prajinkhadka <34759925+prajinkhadka@users.noreply.github.com> Date: Wed, 29 May 2024 23:24:06 +0545 Subject: [PATCH 04/31] Create requirements.txt.3.10 for Python3.10 for Benchmark - 501.graph.pagerank --- .../501.graph-pagerank/python/requirements.txt.3.10 | 1 + 1 file changed, 1 insertion(+) create mode 100644 benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.10 diff --git a/benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.10 b/benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.10 new file mode 100644 index 00000000..e291b7b3 --- /dev/null +++ b/benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.10 @@ -0,0 +1 @@ +igraph==0.11.4 From 0dc5287ad3545f5aad21aac4b19db7b32dca5832 Mon Sep 17 00:00:00 2001 From: prajinkhadka <34759925+prajinkhadka@users.noreply.github.com> Date: Wed, 29 May 2024 23:25:01 +0545 Subject: [PATCH 05/31] Create requirements.txt.3.10 for Python3.10 for Benchmark - 502.graph-mst --- .../500.scientific/502.graph-mst/python/requirements.txt.3.10 | 1 + 1 file changed, 1 insertion(+) create mode 100644 benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.10 diff --git a/benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.10 b/benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.10 new file mode 100644 index 00000000..e291b7b3 --- /dev/null +++ b/benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.10 @@ -0,0 +1 @@ +igraph==0.11.4 From 2e9daf7eae2d13790aca7fc8b176e9232452ff9d Mon Sep 17 00:00:00 2001 From: prajinkhadka <34759925+prajinkhadka@users.noreply.github.com> Date: Wed, 29 May 2024 23:26:04 +0545 Subject: [PATCH 06/31] Create requirements.txt.3.10 for Python 3.10 for Benchmark - 503.graph-bfs --- .../500.scientific/503.graph-bfs/python/requirements.txt.3.10 | 1 + 1 file changed, 1 insertion(+) create mode 100644 benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.10 diff --git a/benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.10 b/benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.10 new file mode 100644 index 00000000..e291b7b3 --- /dev/null +++ b/benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.10 @@ -0,0 +1 @@ +igraph==0.11.4 From dceb50670bb462f0c6c7bcb03a7835b79a8cc266 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Mon, 10 Jun 2024 10:46:25 +0530 Subject: [PATCH 07/31] fix benchmarks run issue for Azure platform Signed-off-by: Abhishek Kumar --- .../411.image-recognition/python/requirements.txt.3.10 | 2 ++ config/systems.json | 9 +++++---- sebs/azure/azure.py | 3 ++- 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.10 diff --git a/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.10 b/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.10 new file mode 100644 index 00000000..71b9e406 --- /dev/null +++ b/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.10 @@ -0,0 +1,2 @@ +pillow==10.3.0 +torchvision==0.18.1 \ No newline at end of file diff --git a/config/systems.json b/config/systems.json index 0c270597..f66204d2 100644 --- a/config/systems.json +++ b/config/systems.json @@ -103,7 +103,8 @@ "base_images": { "3.7": "mcr.microsoft.com/azure-functions/python:3.0-python3.7", "3.8": "mcr.microsoft.com/azure-functions/python:3.0-python3.8", - "3.9": "mcr.microsoft.com/azure-functions/python:3.0-python3.9" + "3.9": "mcr.microsoft.com/azure-functions/python:3.0-python3.9", + "3.10": "mcr.microsoft.com/azure-functions/python:4-python3.10" }, "images": [ "build" @@ -121,9 +122,9 @@ }, "nodejs": { "base_images": { - "16": "mcr.microsoft.com/azure-functions/node:3.0-node16", - "18": "mcr.microsoft.com/azure-functions/node:3.0-node18", - "20": "mcr.microsoft.com/azure-functions/node:3.0-node20" + "16": "mcr.microsoft.com/azure-functions/node:4-node16", + "18": "mcr.microsoft.com/azure-functions/node:4-node18", + "20": "mcr.microsoft.com/azure-functions/node:4-node20" }, "images": [ "build" diff --git a/sebs/azure/azure.py b/sebs/azure/azure.py index 17316c2b..5a1670f4 100644 --- a/sebs/azure/azure.py +++ b/sebs/azure/azure.py @@ -171,7 +171,7 @@ def package_code( "version": "2.0", "extensionBundle": { "id": "Microsoft.Azure.Functions.ExtensionBundle", - "version": "[1.*, 2.0.0)", + "version": "[4.0.0, 5.0.0)", }, } json.dump(default_host_json, open(os.path.join(directory, "host.json"), "w"), indent=2) @@ -335,6 +335,7 @@ def create_function(self, code_package: Benchmark, func_name: str) -> AzureFunct " --os-type Linux --consumption-plan-location {region} " " --runtime {runtime} --runtime-version {runtime_version} " " --name {func_name} --storage-account {storage_account}" + " --functions-version 4 " ).format(**config) ) self.logging.info("Azure: Created function app {}".format(func_name)) From 6f4e024d98bd61689a2cec3a149626968e693e8b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Tue, 11 Jun 2024 14:26:28 +0530 Subject: [PATCH 08/31] fix 210.thumbnailer dependency issue Signed-off-by: Abhishek Kumar --- benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json index 774a1492..aaa3f883 100644 --- a/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json +++ b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json @@ -5,6 +5,6 @@ "author": "", "license": "", "dependencies": { - "sharp": "^0.25" + "sharp": "^0.33" } } From a04fd2bca9cc4e028a823136b429ca4c8ceaa7f0 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 18:21:57 +0200 Subject: [PATCH 09/31] [regression] Exclude 411 for new Python versions on AWS --- sebs/regression.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sebs/regression.py b/sebs/regression.py index 3084bc88..639ff72b 100644 --- a/sebs/regression.py +++ b/sebs/regression.py @@ -126,6 +126,7 @@ def get_deployment(self, benchmark_name): self.client.output_dir, f"regression_{deployment_name}_{benchmark_name}.log" ), ) + with AWSTestSequencePython.lock: deployment_client.initialize(resource_prefix="regression") return deployment_client @@ -322,7 +323,7 @@ def filter_out_benchmarks( benchmark: str, deployment_name: str, language: str, language_version: str ) -> bool: - if deployment_name == "aws" and language == "python" and language_version == "3.9": + if deployment_name == "aws" and language == "python" and language_version in ["3.9", "3.10", "3.11"]: return "411.image-recognition" not in benchmark return True From f553d8ee92ed830f9f04c2e12d5c1079b4daff28 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 18:27:43 +0200 Subject: [PATCH 10/31] [aws] Support Python 3.11 on AWS --- .../210.thumbnailer/python/requirements.txt.3.11 | 1 + .../501.graph-pagerank/python/requirements.txt.3.11 | 1 + .../502.graph-mst/python/requirements.txt.3.11 | 1 + .../503.graph-bfs/python/requirements.txt.3.11 | 1 + config/systems.json | 8 ++++++-- 5 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.11 create mode 100644 benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.11 create mode 100644 benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.11 create mode 100644 benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.11 diff --git a/benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.11 b/benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.11 new file mode 100644 index 00000000..9caa46c8 --- /dev/null +++ b/benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.11 @@ -0,0 +1 @@ +pillow==10.3.0 diff --git a/benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.11 b/benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.11 new file mode 100644 index 00000000..e291b7b3 --- /dev/null +++ b/benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.11 @@ -0,0 +1 @@ +igraph==0.11.4 diff --git a/benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.11 b/benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.11 new file mode 100644 index 00000000..e291b7b3 --- /dev/null +++ b/benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.11 @@ -0,0 +1 @@ +igraph==0.11.4 diff --git a/benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.11 b/benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.11 new file mode 100644 index 00000000..e291b7b3 --- /dev/null +++ b/benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.11 @@ -0,0 +1 @@ +igraph==0.11.4 diff --git a/config/systems.json b/config/systems.json index c082b83e..0ecb5010 100644 --- a/config/systems.json +++ b/config/systems.json @@ -55,13 +55,16 @@ "languages": { "python": { "base_images": { + "3.11": "amazon/aws-lambda-python:3.11", "3.10": "amazon/aws-lambda-python:3.10", "3.9": "amazon/aws-lambda-python:3.9", "3.8": "amazon/aws-lambda-python:3.8" }, "versions": [ "3.8", - "3.9" + "3.9", + "3.10", + "3.12" ], "images": [ "build" @@ -80,7 +83,8 @@ "16": "amazon/aws-lambda-nodejs:16" }, "versions": [ - "14", "16" + "14", + "16" ], "images": [ "build" From 7d76edf72498b0dcccf74609c6679bc0c8a47adb Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 18:55:23 +0200 Subject: [PATCH 11/31] [aws] Deprecate Node.js 14 --- config/systems.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/config/systems.json b/config/systems.json index 0ecb5010..0be683b4 100644 --- a/config/systems.json +++ b/config/systems.json @@ -79,11 +79,9 @@ }, "nodejs": { "base_images": { - "14": "amazon/aws-lambda-nodejs:14", "16": "amazon/aws-lambda-nodejs:16" }, "versions": [ - "14", "16" ], "images": [ From 2731e73df8702cf1e635aa2708405f4c10c616cb Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 20:08:28 +0200 Subject: [PATCH 12/31] [azure] Fix bug in regression - use different directories inside Docker CLI --- sebs/azure/azure.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/sebs/azure/azure.py b/sebs/azure/azure.py index c4924c90..78e45963 100644 --- a/sebs/azure/azure.py +++ b/sebs/azure/azure.py @@ -4,6 +4,7 @@ import os import shutil import time +import uuid from typing import cast, Dict, List, Optional, Set, Tuple, Type # noqa import docker @@ -215,6 +216,7 @@ def publish_function( self, function: Function, code_package: Benchmark, + container_dest: str, repeat_on_failure: bool = False, ) -> str: success = False @@ -223,7 +225,7 @@ def publish_function( while not success: try: ret = self.cli_instance.execute( - "bash -c 'cd /mnt/function " + f"bash -c 'cd {container_dest} " "&& func azure functionapp publish {} --{} --no-build'".format( function.name, self.AZURE_RUNTIMES[code_package.language_name] ) @@ -286,8 +288,8 @@ def publish_function( def update_function(self, function: Function, code_package: Benchmark): # Mount code package in Docker instance - self._mount_function_code(code_package) - url = self.publish_function(function, code_package, True) + container_dest = self._mount_function_code(code_package) + url = self.publish_function(function, code_package, container_dest, True) trigger = HTTPTrigger(url, self.config.resources.data_storage_account(self.cli_instance)) trigger.logging_handlers = self.logging_handlers @@ -299,8 +301,10 @@ def update_function_configuration(self, function: Function, code_package: Benchm "Updating function's memory and timeout configuration is not supported." ) - def _mount_function_code(self, code_package: Benchmark): - self.cli_instance.upload_package(code_package.code_location, "/mnt/function/") + def _mount_function_code(self, code_package: Benchmark) -> str: + dest = os.path.join("/mnt", "function", uuid.uuid4().hex) + self.cli_instance.upload_package(code_package.code_location, dest) + return dest def default_function_name(self, code_package: Benchmark) -> str: """ From a498708eb30d6321bc5a389e388c90515d10d928 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 20:08:38 +0200 Subject: [PATCH 13/31] [azure] Document problem with large archives --- sebs/azure/cli.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sebs/azure/cli.py b/sebs/azure/cli.py index 96ccf65e..af718f33 100644 --- a/sebs/azure/cli.py +++ b/sebs/azure/cli.py @@ -85,10 +85,24 @@ def login(self, appId: str, tenant: str, password: str) -> bytes: return result def upload_package(self, directory: str, dest: str): + + """ + This is not an efficient and memory-intensive implementation. + So far, we didn't have very large functions that require many gigabytes. + + Since docker-py does not support a straightforward copy, and we can't + put_archive in chunks. + + If we end up having problems because of the archive size, there are two + potential solutions: + (1) manually call docker cp and decompress + (2) commit the docker container and restart with a new mount volume. + """ handle = io.BytesIO() with tarfile.open(fileobj=handle, mode="w:gz") as tar: for f in os.listdir(directory): tar.add(os.path.join(directory, f), arcname=f) + # shutil.make_archive(, 'zip', directory) # move to the beginning of memory before writing handle.seek(0) self.execute("mkdir -p {}".format(dest)) From bbe0873903a4277daeed28e34d6d8566fcdde6a5 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 20:35:41 +0200 Subject: [PATCH 14/31] [benchmarks] Update PyTorch inference to Python 3.10 --- .../411.image-recognition/python/requirements.txt.3.10 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.10 b/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.10 index 71b9e406..ab734881 100644 --- a/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.10 +++ b/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.10 @@ -1,2 +1,5 @@ pillow==10.3.0 -torchvision==0.18.1 \ No newline at end of file +https://download.pytorch.org/whl/cpu/torch-1.11.0%2Bcpu-cp310-cp310-linux_x86_64.whl +torchvision==0.12 +# prevent installing numpy 2.0 +numpy==1.22.0 From 5e8aefd262742aa2ac1619b79c43055b049b7f36 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 21:08:28 +0200 Subject: [PATCH 15/31] [regression] Fix excluding 411 benchmark for aws and gcp --- sebs/regression.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sebs/regression.py b/sebs/regression.py index 639ff72b..a3eae7eb 100644 --- a/sebs/regression.py +++ b/sebs/regression.py @@ -323,7 +323,12 @@ def filter_out_benchmarks( benchmark: str, deployment_name: str, language: str, language_version: str ) -> bool: - if deployment_name == "aws" and language == "python" and language_version in ["3.9", "3.10", "3.11"]: + if (deployment_name == "aws" and language == "python" + and language_version in ["3.9", "3.10", "3.11"]): + return "411.image-recognition" not in benchmark + + if (deployment_name == "gcp" and language == "python" + and language_version in ["3.8", "3.9", "3.10", "3.11", "3.12"]): return "411.image-recognition" not in benchmark return True From 52097e80914d688fb922e59db1fdbed838039e62 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 21:08:54 +0200 Subject: [PATCH 16/31] [benchmarks] Upgrade 411.image-recognition to Python 3.11 --- .../411.image-recognition/python/requirements.txt.3.11 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.11 diff --git a/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.11 b/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.11 new file mode 100644 index 00000000..3288171f --- /dev/null +++ b/benchmarks/400.inference/411.image-recognition/python/requirements.txt.3.11 @@ -0,0 +1,5 @@ +pillow==10.3.0 +https://download.pytorch.org/whl/cpu/torch-2.0.0%2Bcpu-cp311-cp311-linux_x86_64.whl +torchvision==0.15.1 +# prevent installing numpy 2.0 +numpy==1.24.0 From 598c4ced3d322e7c1aedf07b322bc8949170a566 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 21:58:33 +0200 Subject: [PATCH 17/31] [benchmarks] Support 210.thumbnailer on Node.js 16, 18 & 20 --- .../200.multimedia/210.thumbnailer/nodejs/package.json | 2 +- .../210.thumbnailer/nodejs/package.json.18 | 10 ++++++++++ .../210.thumbnailer/nodejs/package.json.20 | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.18 create mode 100644 benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.20 diff --git a/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json index aaa3f883..774a1492 100644 --- a/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json +++ b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json @@ -5,6 +5,6 @@ "author": "", "license": "", "dependencies": { - "sharp": "^0.33" + "sharp": "^0.25" } } diff --git a/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.18 b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.18 new file mode 100644 index 00000000..aaa3f883 --- /dev/null +++ b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.18 @@ -0,0 +1,10 @@ +{ + "name": "", + "version": "1.0.0", + "description": "", + "author": "", + "license": "", + "dependencies": { + "sharp": "^0.33" + } +} diff --git a/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.20 b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.20 new file mode 100644 index 00000000..aaa3f883 --- /dev/null +++ b/benchmarks/200.multimedia/210.thumbnailer/nodejs/package.json.20 @@ -0,0 +1,10 @@ +{ + "name": "", + "version": "1.0.0", + "description": "", + "author": "", + "license": "", + "dependencies": { + "sharp": "^0.33" + } +} From 7c77e2c6dbadd354488bafa6e6b2b94a94ec529b Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 21:59:00 +0200 Subject: [PATCH 18/31] [docs] Update info about Python versions on Google Cloud --- docs/benchmarks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 2d913607..85a8f969 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -26,7 +26,7 @@ Below, we discuss the most important implementation details of each benchmark. F > Benchmark `411.image-recognition` does not work on AWS with Python 3.9 due to excessive code size. While it is possible to ship the benchmark by zipping `torchvision` and `numpy` (see `benchmarks/400.inference/411.image-recognition/python/package.sh`), this significantly affects cold startup. On the lowest supported memory configuration of 512 MB, the cold startup can reach 30 seconds, making HTTP trigger unusable due to 30 second timeout of API gateway. In future, we might support Docker-based deployment on AWS that are not affected by code size limitations. > [!WARNING] -> Benchmark `411.image-recognition` does not work on GCP with Python 3.8 and 3.9 due to excessive code size. To the best of our knowledge, there is no way of circumventing that limit, as Google Cloud offers neither layers nor custom Docker images. +> Benchmark `411.image-recognition` does not work on GCP with Python 3.8+ due to excessive code size. To the best of our knowledge, there is no way of circumventing that limit, as Google Cloud offers neither layers nor custom Docker images. ## Webapps From 587c1a1df0449d8b17cfcf4cc537e72a1ea5606c Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 21:59:43 +0200 Subject: [PATCH 19/31] [azure] Add documentation on the problematic upload of archive to Docker --- sebs/azure/cli.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/sebs/azure/cli.py b/sebs/azure/cli.py index af718f33..b875ee02 100644 --- a/sebs/azure/cli.py +++ b/sebs/azure/cli.py @@ -87,16 +87,16 @@ def login(self, appId: str, tenant: str, password: str) -> bytes: def upload_package(self, directory: str, dest: str): """ - This is not an efficient and memory-intensive implementation. - So far, we didn't have very large functions that require many gigabytes. + This is not an efficient and memory-intensive implementation. + So far, we didn't have very large functions that require many gigabytes. - Since docker-py does not support a straightforward copy, and we can't - put_archive in chunks. + Since docker-py does not support a straightforward copy, and we can't + put_archive in chunks. - If we end up having problems because of the archive size, there are two - potential solutions: - (1) manually call docker cp and decompress - (2) commit the docker container and restart with a new mount volume. + If we end up having problems because of the archive size, there are two + potential solutions: + (1) manually call docker cp and decompress + (2) commit the docker container and restart with a new mount volume. """ handle = io.BytesIO() with tarfile.open(fileobj=handle, mode="w:gz") as tar: From ea153a4ed1537f1b0775d3e4c207bcab20ee0782 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 22:17:06 +0200 Subject: [PATCH 20/31] [regression] Use single deployment for gcp --- sebs/regression.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sebs/regression.py b/sebs/regression.py index a3eae7eb..71012284 100644 --- a/sebs/regression.py +++ b/sebs/regression.py @@ -230,7 +230,8 @@ def get_deployment(self, benchmark_name): cloud_config, logging_filename=f"regression_{deployment_name}_{benchmark_name}.log", ) - deployment_client.initialize() + with GCPTestSequencePython.lock: + deployment_client.initialize(resource_prefix="regression") return deployment_client @@ -248,7 +249,8 @@ def get_deployment(self, benchmark_name): cloud_config, logging_filename=f"regression_{deployment_name}_{benchmark_name}.log", ) - deployment_client.initialize() + with GCPTestSequenceNodejs.lock: + deployment_client.initialize(resource_prefix="regression") return deployment_client From 8ee7b80b3913cc7309ab0eb5d468ed9d1ed12fd4 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Tue, 9 Jul 2024 23:59:55 +0200 Subject: [PATCH 21/31] [benchmarks] Update benchmarks for Python 3.12 --- .../200.multimedia/210.thumbnailer/python/requirements.txt.3.12 | 1 + .../501.graph-pagerank/python/requirements.txt.3.12 | 1 + .../500.scientific/502.graph-mst/python/requirements.txt.3.12 | 1 + .../500.scientific/503.graph-bfs/python/requirements.txt.3.12 | 1 + 4 files changed, 4 insertions(+) create mode 100644 benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.12 create mode 100644 benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.12 create mode 100644 benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.12 create mode 100644 benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.12 diff --git a/benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.12 b/benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.12 new file mode 100644 index 00000000..9caa46c8 --- /dev/null +++ b/benchmarks/200.multimedia/210.thumbnailer/python/requirements.txt.3.12 @@ -0,0 +1 @@ +pillow==10.3.0 diff --git a/benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.12 b/benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.12 new file mode 100644 index 00000000..e291b7b3 --- /dev/null +++ b/benchmarks/500.scientific/501.graph-pagerank/python/requirements.txt.3.12 @@ -0,0 +1 @@ +igraph==0.11.4 diff --git a/benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.12 b/benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.12 new file mode 100644 index 00000000..e291b7b3 --- /dev/null +++ b/benchmarks/500.scientific/502.graph-mst/python/requirements.txt.3.12 @@ -0,0 +1 @@ +igraph==0.11.4 diff --git a/benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.12 b/benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.12 new file mode 100644 index 00000000..e291b7b3 --- /dev/null +++ b/benchmarks/500.scientific/503.graph-bfs/python/requirements.txt.3.12 @@ -0,0 +1 @@ +igraph==0.11.4 From 8713e9e4e2ce9d22e8f952426a7663be196992bf Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 00:00:14 +0200 Subject: [PATCH 22/31] [gcp] Update supported versions --- config/systems.json | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/config/systems.json b/config/systems.json index 0be683b4..19a26e2b 100644 --- a/config/systems.json +++ b/config/systems.json @@ -60,12 +60,6 @@ "3.9": "amazon/aws-lambda-python:3.9", "3.8": "amazon/aws-lambda-python:3.8" }, - "versions": [ - "3.8", - "3.9", - "3.10", - "3.12" - ], "images": [ "build" ], @@ -81,9 +75,6 @@ "base_images": { "16": "amazon/aws-lambda-nodejs:16" }, - "versions": [ - "16" - ], "images": [ "build" ], @@ -106,7 +97,8 @@ "3.7": "mcr.microsoft.com/azure-functions/python:3.0-python3.7", "3.8": "mcr.microsoft.com/azure-functions/python:3.0-python3.8", "3.9": "mcr.microsoft.com/azure-functions/python:3.0-python3.9", - "3.10": "mcr.microsoft.com/azure-functions/python:4-python3.10" + "3.10": "mcr.microsoft.com/azure-functions/python:4-python3.10", + "3.11": "mcr.microsoft.com/azure-functions/python:4-python3.11" }, "images": [ "build" @@ -155,7 +147,9 @@ "python": { "base_images": { "3.9": "ubuntu:22.04", - "3.10": "ubuntu:22.04" + "3.10": "ubuntu:22.04", + "3.11": "ubuntu:22.04", + "3.12": "ubuntu:22.04" }, "images": [ "build" @@ -173,6 +167,10 @@ }, "nodejs": { "base_images": { + "10": "gcr.io/google-appengine/nodejs", + "12": "gcr.io/google-appengine/nodejs", + "14": "gcr.io/google-appengine/nodejs", + "16": "gcr.io/google-appengine/nodejs", "18": "gcr.io/google-appengine/nodejs", "20": "gcr.io/google-appengine/nodejs" }, From de412d4006df0f50342a6c9c131635ab70e3dfc3 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 01:17:32 +0200 Subject: [PATCH 23/31] [gcp] Update Docker images to work with Node.js 18 and 20 --- config/systems.json | 13 +++++++------ dockerfiles/gcp/nodejs/Dockerfile.build | 9 ++++++--- dockerfiles/nodejs_installer.sh | 3 +++ 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/config/systems.json b/config/systems.json index 19a26e2b..a73d5ac6 100644 --- a/config/systems.json +++ b/config/systems.json @@ -146,6 +146,7 @@ "languages": { "python": { "base_images": { + "3.8": "ubuntu:22.04", "3.9": "ubuntu:22.04", "3.10": "ubuntu:22.04", "3.11": "ubuntu:22.04", @@ -167,12 +168,12 @@ }, "nodejs": { "base_images": { - "10": "gcr.io/google-appengine/nodejs", - "12": "gcr.io/google-appengine/nodejs", - "14": "gcr.io/google-appengine/nodejs", - "16": "gcr.io/google-appengine/nodejs", - "18": "gcr.io/google-appengine/nodejs", - "20": "gcr.io/google-appengine/nodejs" + "10": "ubuntu:18.04", + "12": "ubuntu:18.04", + "14": "ubuntu:18.04", + "16": "ubuntu:18.04", + "18": "ubuntu:22.04", + "20": "ubuntu:22.04" }, "images": [ "build" diff --git a/dockerfiles/gcp/nodejs/Dockerfile.build b/dockerfiles/gcp/nodejs/Dockerfile.build index fbc4e0bd..477f236b 100755 --- a/dockerfiles/gcp/nodejs/Dockerfile.build +++ b/dockerfiles/gcp/nodejs/Dockerfile.build @@ -1,10 +1,12 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} ARG VERSION -ENV HOME=/home/${USER} +ENV NVM_DIR=/nvm -RUN install_node --ignore-verification-failure v${VERSION} -RUN apt-get update && apt-get install -y gosu +#RUN install_node --ignore-verification-failure v${VERSION} +RUN apt-get update && apt-get install -y gosu wget +RUN mkdir -p ${NVM_DIR} && wget -qO- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash +RUN . ${NVM_DIR}/nvm.sh && nvm install ${VERSION} && nvm alias default ${VERSION} && nvm use default RUN mkdir -p /sebs/ COPY dockerfiles/nodejs_installer.sh /sebs/installer.sh @@ -16,3 +18,4 @@ ENV PATH=/usr/sbin:$PATH ENV SCRIPT_FILE=/mnt/function/package.sh CMD /bin/bash /sebs/installer.sh ENTRYPOINT ["/sebs/entrypoint.sh"] + diff --git a/dockerfiles/nodejs_installer.sh b/dockerfiles/nodejs_installer.sh index 7f0710f3..d6fd6a5b 100644 --- a/dockerfiles/nodejs_installer.sh +++ b/dockerfiles/nodejs_installer.sh @@ -1,3 +1,6 @@ #!/bin/bash +if [ -f $FILE ]; then + . /nvm/nvm.sh +fi cd /mnt/function && npm install && rm -rf package-lock.json From 8cdff2ddb0574d0b5eff2c3af30011625218b989 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 01:52:50 +0200 Subject: [PATCH 24/31] [openwhisk] Update Openwhisk images --- config/systems.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/systems.json b/config/systems.json index a73d5ac6..a8fc7326 100644 --- a/config/systems.json +++ b/config/systems.json @@ -217,6 +217,9 @@ }, "nodejs": { "base_images": { + "10": "openwhisk/action-nodejs-v10", + "12": "openwhisk/action-nodejs-v12", + "14": "openwhisk/action-nodejs-v14", "18": "openwhisk/action-nodejs-v18", "20": "openwhisk/action-nodejs-v20" }, From 917e2b4d3af8f1ba53d740ca57b5b7d8dd8ad37d Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 02:00:48 +0200 Subject: [PATCH 25/31] [openwhisk] Revert old Python version --- config/systems.json | 1 + 1 file changed, 1 insertion(+) diff --git a/config/systems.json b/config/systems.json index a8fc7326..2cdb56c1 100644 --- a/config/systems.json +++ b/config/systems.json @@ -196,6 +196,7 @@ "languages": { "python": { "base_images": { + "3.7": "openwhisk/action-python-v3.7", "3.9": "openwhisk/action-python-v3.9", "3.10": "openwhisk/action-python-v3.10", "3.11": "openwhisk/action-python-v3.11" From 86920aa507188465cbd3985a54551a42a9ffd3e6 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 02:02:36 +0200 Subject: [PATCH 26/31] [gcp] Revert older Python version --- config/systems.json | 1 + 1 file changed, 1 insertion(+) diff --git a/config/systems.json b/config/systems.json index 2cdb56c1..c8f3720e 100644 --- a/config/systems.json +++ b/config/systems.json @@ -146,6 +146,7 @@ "languages": { "python": { "base_images": { + "3.7": "ubuntu:22.04", "3.8": "ubuntu:22.04", "3.9": "ubuntu:22.04", "3.10": "ubuntu:22.04", From 82b66c41d5ccbf3e0e1d777028b358e7992c6b0d Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 02:41:59 +0200 Subject: [PATCH 27/31] [local] Add GET method to control if server is alive --- dockerfiles/local/python/server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dockerfiles/local/python/server.py b/dockerfiles/local/python/server.py index e86327dc..4ed1314f 100644 --- a/dockerfiles/local/python/server.py +++ b/dockerfiles/local/python/server.py @@ -8,8 +8,14 @@ CODE_LOCATION='/function' +@route('/alive', method='GET') +def alive(): + return { + "result:" "ok" + } + @route('/', method='POST') -def flush_log(): +def process_request(): begin = datetime.datetime.now() from function import function end = datetime.datetime.now() From 131145fcd705db255a1ede73e280065b9e301254 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 02:42:33 +0200 Subject: [PATCH 28/31] [local] Wait until container is running --- requirements.txt | 1 + sebs/local/function.py | 4 ++++ sebs/local/local.py | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/requirements.txt b/requirements.txt index cd53509b..2717467c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ testtools>=2.4.0 docker>=4.2.0 tzlocal>=2.1 +requests #linting flake8 black==22.8.0 diff --git a/sebs/local/function.py b/sebs/local/function.py index a5eb2406..1db9d4cb 100644 --- a/sebs/local/function.py +++ b/sebs/local/function.py @@ -67,6 +67,10 @@ def __init__( self._measurement_pid = measurement_pid + @property + def url(self) -> str: + return self._url + @property def memory_measurement_pid(self) -> Optional[int]: return self._measurement_pid diff --git a/sebs/local/local.py b/sebs/local/local.py index fb9bd1e1..5a4eb18f 100644 --- a/sebs/local/local.py +++ b/sebs/local/local.py @@ -1,5 +1,7 @@ import os +import requests import shutil +import time from typing import cast, Dict, List, Optional, Type, Tuple # noqa import subprocess @@ -221,6 +223,24 @@ def create_function(self, code_package: Benchmark, func_name: str) -> "LocalFunc function_cfg, pid, ) + + # Wait until server starts + max_attempts = 10 + attempts = 0 + while attempts < max_attempts: + try: + requests.get(f"http://{func.url}/alive") + break + except requests.exceptions.ConnectionError: + time.sleep(0.25) + attempts += 1 + + if attempts == max_attempts: + raise RuntimeError( + f"Couldn't start {func_name} function at container " + f"{container.id} , running on {func._url}" + ) + self.logging.info( f"Started {func_name} function at container {container.id} , running on {func._url}" ) From 6d05bb2d017b49a7ebaaa6768dcc2bb5497caf9b Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 02:43:59 +0200 Subject: [PATCH 29/31] [local] Adapt the build image to Python3 --- dockerfiles/local/python/Dockerfile.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dockerfiles/local/python/Dockerfile.build b/dockerfiles/local/python/Dockerfile.build index 9d6402af..5892c650 100755 --- a/dockerfiles/local/python/Dockerfile.build +++ b/dockerfiles/local/python/Dockerfile.build @@ -4,7 +4,7 @@ ARG VERSION ENV PYTHON_VERSION=${VERSION} RUN apt-get update\ - && apt-get install -y --no-install-recommends gcc build-essential python-dev libxml2 libxml2-dev zlib1g-dev gosu\ + && apt-get install -y --no-install-recommends gcc build-essential python3-dev libxml2 libxml2-dev zlib1g-dev gosu\ && apt-get purge -y --auto-remove RUN mkdir -p /sebs/ From 8cb9807b8f8b157b7ba1d1d440e7976bd5a64783 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 02:59:51 +0200 Subject: [PATCH 30/31] [local] Update supported languge versions --- config/systems.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/config/systems.json b/config/systems.json index c8f3720e..bf095d3f 100644 --- a/config/systems.json +++ b/config/systems.json @@ -17,8 +17,11 @@ "languages": { "python": { "base_images": { + "3.7": "python:3.7-slim", "3.8": "python:3.8-slim", - "3.9": "python:3.9-slim" + "3.9": "python:3.9-slim", + "3.10": "python:3.10-slim", + "3.11": "python:3.11-slim" }, "images": [ "run", @@ -34,6 +37,8 @@ }, "nodejs": { "base_images": { + "14": "node:14-slim", + "16": "node:16-slim", "18": "node:18-slim", "20": "node:20-slim" }, From e0c29158f1cb9c9b3548a0565d541142e93081d8 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Wed, 10 Jul 2024 03:00:52 +0200 Subject: [PATCH 31/31] [local] Extend node.js server with a GET alive method --- dockerfiles/local/nodejs/server.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dockerfiles/local/nodejs/server.js b/dockerfiles/local/nodejs/server.js index b40696d7..c98b3fa7 100644 --- a/dockerfiles/local/nodejs/server.js +++ b/dockerfiles/local/nodejs/server.js @@ -9,6 +9,12 @@ const { v4: uuidv4 } = require('uuid'); var app = express(); app.use(express.json()); +app.post('/alive', function (req, res) { + res.send(JSON.stringify({ + status: "ok" + })); +}); + app.post('/', function (req, res) { let begin = Date.now();