diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 3c174931e0f..bb66d859dee 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -4,7 +4,7 @@ on: push: branches: ["*", "**"] paths-ignore: [ '**.md', '**.png', '**.jpg', '**.svg', '**/docs/**' ] - pull_request: + pull_request_target: branches: ["*", "**"] paths-ignore: [ '**.md', '**.png', '**.jpg', '**.svg', '**/docs/**' ] schedule: @@ -14,6 +14,10 @@ on: env: CARGO_TERM_COLOR: always + SNAPSHOOTER_VERSION: 0.7.3 + NERDCTL_VERSION: 1.3.0 + CNI_PLUGINS_VERSION: 1.2.0 + IMAGE: wordpress jobs: contrib-build: @@ -128,6 +132,337 @@ jobs: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.51.2 sudo -E make smoke-only + benchmark-oci: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download Nydus + uses: actions/download-artifact@master + with: + name: nydus-artifact + path: target/release + - name: Download Nydusify + uses: actions/download-artifact@master + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Nerdctl Environment + run: | + sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/local/bin + sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/local/bin + sudo wget https://github.com/containerd/nerdctl/releases/download/v${{env.NERDCTL_VERSION}}/nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz + sudo tar -xzvf nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz -C /usr/local/bin + sudo mkdir -p /opt/cni/bin + sudo wget https://github.com/containernetworking/plugins/releases/download/v${{env.CNI_PLUGINS_VERSION}}/cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz + sudo tar -xzvf cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz -C /opt/cni/bin + sudo install -D misc/benchmark/cni_bridge.conf /etc/cni/net.d/bridge.conf + - name: Prepare OCI Registry Environment + run: | + sudo docker run -d --restart=always -p 5000:5000 --name registry registry + sudo docker pull ${{env.IMAGE}} && docker tag ${{env.IMAGE}}:latest localhost:5000/${{env.IMAGE}}:latest + sudo docker push localhost:5000/${{env.IMAGE}}:latest + git clone https://github.com/magnific0/wondershaper.git + sudo install -D -m 755 wondershaper/wondershaper /usr/local/bin + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode oci + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-oci + path: misc/benchmark/${{env.IMAGE}}.csv + + benchmark-nydus-no-prefetch: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download Nydus + uses: actions/download-artifact@master + with: + name: nydus-artifact + path: target/release + - name: Download Nydusify + uses: actions/download-artifact@master + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Nydus Environment + run: | + sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/local/bin + sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/local/bin + wget https://github.com/containerd/nydus-snapshotter/releases/download/v${{env.SNAPSHOOTER_VERSION}}/nydus-snapshotter-v${{env.SNAPSHOOTER_VERSION}}-x86_64.tgz + tar zxvf nydus-snapshotter-v${{env.SNAPSHOOTER_VERSION}}-x86_64.tgz + sudo install -D -m 755 nydus-snapshotter/containerd-nydus-grpc /usr/local/bin/ + sudo wget https://github.com/containerd/nerdctl/releases/download/v${{env.NERDCTL_VERSION}}/nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz + sudo tar -xzvf nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz -C /usr/local/bin + sudo mkdir -p /opt/cni/bin + sudo wget https://github.com/containernetworking/plugins/releases/download/v${{env.CNI_PLUGINS_VERSION}}/cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz + sudo tar -xzvf cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz -C /opt/cni/bin + sudo install -D misc/benchmark/cni_bridge.conf /etc/cni/net.d/bridge.conf + sudo install -D misc/benchmark/nydusd_config.json /etc/nydus/config.json + sudo install -D misc/benchmark/containerd_config.toml /etc/containerd/config.toml + sudo systemctl restart containerd + sudo install -D misc/benchmark/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service + sudo systemctl start nydus-snapshotter + - name: Prepare Nydus Registry Environment + run: | + sudo docker run -d --restart=always -p 5000:5000 --name registry registry + sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ + --source ${{env.IMAGE}}:latest \ + --target localhost:5000/${{env.IMAGE}}:latest_nydus \ + --fs-version 6 \ + --platform linux/amd64,linux/arm64 + git clone https://github.com/magnific0/wondershaper.git + sudo install -D -m 755 wondershaper/wondershaper /usr/local/bin + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode nydus-no-prefetch + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-nydus-no-prefetch + path: misc/benchmark/${{env.IMAGE}}.csv + + benchmark-nydus-all-prefetch: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download Nydus + uses: actions/download-artifact@master + with: + name: nydus-artifact + path: target/release + - name: Download Nydusify + uses: actions/download-artifact@master + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Nydus Environment + run: | + sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/local/bin + sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/local/bin + wget https://github.com/containerd/nydus-snapshotter/releases/download/v${{env.SNAPSHOOTER_VERSION}}/nydus-snapshotter-v${{env.SNAPSHOOTER_VERSION}}-x86_64.tgz + tar zxvf nydus-snapshotter-v${{env.SNAPSHOOTER_VERSION}}-x86_64.tgz + sudo install -D -m 755 nydus-snapshotter/containerd-nydus-grpc /usr/local/bin/ + sudo wget https://github.com/containerd/nerdctl/releases/download/v${{env.NERDCTL_VERSION}}/nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz + sudo tar -xzvf nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz -C /usr/local/bin + sudo mkdir -p /opt/cni/bin + sudo wget https://github.com/containernetworking/plugins/releases/download/v${{env.CNI_PLUGINS_VERSION}}/cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz + sudo tar -xzvf cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz -C /opt/cni/bin + sudo install -D misc/benchmark/cni_bridge.conf /etc/cni/net.d/bridge.conf + sudo install -D misc/benchmark/nydusd_config.json /etc/nydus/config.json + sudo install -D misc/benchmark/containerd_config.toml /etc/containerd/config.toml + sudo systemctl restart containerd + sudo install -D misc/benchmark/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service + sudo systemctl start nydus-snapshotter + - name: Prepare Nydus Registry Environment + run: | + sudo docker run -d --restart=always -p 5000:5000 --name registry registry + sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ + --source ${{env.IMAGE}}:latest \ + --target localhost:5000/${{env.IMAGE}}:latest_nydus \ + --fs-version 6 \ + --platform linux/amd64,linux/arm64 + git clone https://github.com/magnific0/wondershaper.git + sudo install -D -m 755 wondershaper/wondershaper /usr/local/bin + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode nydus-all-prefetch + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-nydus-all-prefetch + path: misc/benchmark/${{env.IMAGE}}.csv + + benchmark-zran-all-prefetch: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download Nydus + uses: actions/download-artifact@master + with: + name: nydus-artifact + path: target/release + - name: Download Nydusify + uses: actions/download-artifact@master + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Nydus Environment + run: | + sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/local/bin + sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/local/bin + wget https://github.com/containerd/nydus-snapshotter/releases/download/v${{env.SNAPSHOOTER_VERSION}}/nydus-snapshotter-v${{env.SNAPSHOOTER_VERSION}}-x86_64.tgz + tar zxvf nydus-snapshotter-v${{env.SNAPSHOOTER_VERSION}}-x86_64.tgz + sudo install -D -m 755 nydus-snapshotter/containerd-nydus-grpc /usr/local/bin/ + sudo wget https://github.com/containerd/nerdctl/releases/download/v${{env.NERDCTL_VERSION}}/nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz + sudo tar -xzvf nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz -C /usr/local/bin + sudo mkdir -p /opt/cni/bin + sudo wget https://github.com/containernetworking/plugins/releases/download/v${{env.CNI_PLUGINS_VERSION}}/cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz + sudo tar -xzvf cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz -C /opt/cni/bin + sudo install -D misc/benchmark/cni_bridge.conf /etc/cni/net.d/bridge.conf + sudo install -D misc/benchmark/nydusd_config.json /etc/nydus/config.json + sudo install -D misc/benchmark/containerd_config.toml /etc/containerd/config.toml + sudo systemctl restart containerd + sudo install -D misc/benchmark/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service + sudo systemctl start nydus-snapshotter + - name: Prepare Nydus Registry Environment + run: | + sudo docker run -d --restart=always -p 5000:5000 --name registry registry + sudo docker pull ${{env.IMAGE}} && docker tag ${{env.IMAGE}}:latest localhost:5000/${{env.IMAGE}}:latest + sudo docker push localhost:5000/${{env.IMAGE}}:latest + sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ + --source localhost:5000/${{env.IMAGE}}:latest \ + --target localhost:5000/${{env.IMAGE}}:latest_nydus \ + --fs-version 6 \ + --oci-ref \ + --platform linux/amd64,linux/arm64 + git clone https://github.com/magnific0/wondershaper.git + sudo install -D -m 755 wondershaper/wondershaper /usr/local/bin + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode nydus-all-prefetch + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-zran-all-prefetch + path: misc/benchmark/${{env.IMAGE}}.csv + + benchmark-nydus-filelist-prefetch: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download Nydus + uses: actions/download-artifact@master + with: + name: nydus-artifact + path: target/release + - name: Download Nydusify + uses: actions/download-artifact@master + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Nydus Environment + run: | + sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/local/bin + sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/local/bin + wget https://github.com/containerd/nydus-snapshotter/releases/download/v${{env.SNAPSHOOTER_VERSION}}/nydus-snapshotter-v${{env.SNAPSHOOTER_VERSION}}-x86_64.tgz + tar zxvf nydus-snapshotter-v${{env.SNAPSHOOTER_VERSION}}-x86_64.tgz + sudo install -D -m 755 nydus-snapshotter/containerd-nydus-grpc /usr/local/bin/ + sudo wget https://github.com/containerd/nerdctl/releases/download/v${{env.NERDCTL_VERSION}}/nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz + sudo tar -xzvf nerdctl-${{env.NERDCTL_VERSION}}-linux-amd64.tar.gz -C /usr/local/bin + sudo mkdir -p /opt/cni/bin + sudo wget https://github.com/containernetworking/plugins/releases/download/v${{env.CNI_PLUGINS_VERSION}}/cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz + sudo tar -xzvf cni-plugins-linux-amd64-v${{env.CNI_PLUGINS_VERSION}}.tgz -C /opt/cni/bin + sudo install -D misc/benchmark/cni_bridge.conf /etc/cni/net.d/bridge.conf + sudo install -D misc/benchmark/nydusd_config.json /etc/nydus/config.json + sudo install -D misc/benchmark/containerd_config.toml /etc/containerd/config.toml + sudo systemctl restart containerd + sudo install -D misc/benchmark/nydus-snapshotter.service /etc/systemd/system/nydus-snapshotter.service + sudo systemctl start nydus-snapshotter + - name: Prepare Nydus Registry Environment + run: | + sudo docker run -d --restart=always -p 5000:5000 --name registry registry + sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \ + --source ${{env.IMAGE}}:latest \ + --target localhost:5000/${{env.IMAGE}}:latest_nydus \ + --fs-version 6 \ + --platform linux/amd64,linux/arm64 + git clone https://github.com/magnific0/wondershaper.git + sudo install -D -m 755 wondershaper/wondershaper /usr/local/bin + - name: BenchMark Test + run: | + cd misc/benchmark + sudo python3 benchmark.py --mode nydus-filelist-prefetch + - name: Save Test Result + uses: actions/upload-artifact@v3 + with: + name: benchmark-nydus-filelist-prefetch + path: misc/benchmark/${{env.IMAGE}}.csv + + benchmark-result: + runs-on: ubuntu-latest + if: github.event_name == 'pull_request_target' + needs: [benchmark-oci, benchmark-zran-all-prefetch, benchmark-nydus-no-prefetch, benchmark-nydus-all-prefetch, benchmark-nydus-filelist-prefetch] + steps: + - name: Download benchmark-oci + uses: actions/download-artifact@master + with: + name: benchmark-oci + path: benchmark-oci + - name: Download benchmark-zran-all-prefetch + uses: actions/download-artifact@master + with: + name: benchmark-zran-all-prefetch + path: benchmark-zran-all-prefetch + - name: Download benchmark-nydus-no-prefetch + uses: actions/download-artifact@master + with: + name: benchmark-nydus-no-prefetch + path: benchmark-nydus-no-prefetch + - name: Download benchmark-nydus-all-prefetch + uses: actions/download-artifact@master + with: + name: benchmark-nydus-all-prefetch + path: benchmark-nydus-all-prefetch + - name: Download benchmark-nydus-filelist-prefetch + uses: actions/download-artifact@master + with: + name: benchmark-nydus-filelist-prefetch + path: benchmark-nydus-filelist-prefetch + - name: Save Result + run: | + sudo install -m 755 benchmark-oci/wordpress.csv oci.csv + sudo install -m 755 benchmark-zran-all-prefetch/wordpress.csv zran-all-prefetch.csv + sudo install -m 755 benchmark-nydus-no-prefetch/wordpress.csv nydus-no-prefetch.csv + sudo install -m 755 benchmark-nydus-all-prefetch/wordpress.csv nydus-all-prefetch.csv + sudo install -m 755 benchmark-nydus-filelist-prefetch/wordpress.csv nydus-filelist-prefetch.csv + + echo "| benchmark-result | pull-elapsed(s) | create-elapsed(s) | run-elapsed(s) | total-elapsed(s) |" > result.md + echo "|:-------|:-----------------:|:-------------------:|:----------------:|:------------------:|" >> result.md + + for file in *.csv; do + if ! [ -f "$file" ]; then + continue + fi + filename=$(basename "$file" .csv) + tail -n +2 "$file" | while read line; do + pull=$(echo "$line" | cut -d ',' -f 2) + create=$(echo "$line" | cut -d ',' -f 3) + run=$(echo "$line" | cut -d ',' -f 4) + total=$(echo "$line" | cut -d ',' -f 5) + printf "| %s | %s | %s | %s | %s |\n" "$filename" "$pull" "$create" "$run" "$total" + done >> result.md + done + - name: Comment + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const markdownFilePath = './result.md'; + const markdownContent = fs.readFileSync(markdownFilePath, 'utf8'); + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: markdownContent + }) + nydus-unit-test: runs-on: ubuntu-latest steps: diff --git a/misc/benchmark/bench.py b/misc/benchmark/bench.py new file mode 100644 index 00000000000..f6f2f15e148 --- /dev/null +++ b/misc/benchmark/bench.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +""" + bench.py references the repo[https://github.com/nydusaccelerator/hello-bench]. +""" +import copy +import json +import logging +import os +import posixpath +import subprocess +import sys +import time +import urllib.request +from contextlib import contextmanager +from datetime import datetime +from io import TextIOWrapper + + +def logging_setup(logging_stream=sys.stderr): + root = logging.getLogger() + + if root.hasHandlers(): + return + + verbose = True + + handler = logging.StreamHandler(logging_stream) + + if verbose: + root.setLevel(logging.DEBUG) + handler.setLevel(logging.DEBUG) + else: + root.setLevel(logging.INFO) + handler.setLevel(logging.INFO) + + formatter = logging.Formatter( + "[%(asctime)s] %(levelname)s " + "[%(module)s - %(lineno)s:%(funcName)s] " + "- %(message)s" + ) + handler.setFormatter(formatter) + root.addHandler(handler) + + +logging_setup() + + +def run(cmd, wait: bool = True, verbose=True, **kwargs): + + shell = kwargs.pop("shell", False) + if shell: + cmd = " ".join(cmd) + + if verbose: + logging.info(cmd) + else: + logging.debug(cmd) + + popen_obj = subprocess.Popen(cmd, shell=shell, **kwargs) + if wait: + popen_obj.wait() + return popen_obj.returncode, popen_obj + + +def get_current_time(): + return datetime.now() + + +def delta_time(t_end, t_start): + delta = t_end - t_start + return delta.total_seconds(), delta.microseconds + + +@contextmanager +def timer(cmd): + start = get_current_time() + try: + rc = os.system(cmd) + assert rc == 0 + end = get_current_time() + sec, usec = delta_time(end, start) + yield sec + usec / 1e6 + logging.info("%s, Takes time %u.%u seconds", cmd, sec, usec) + finally: + pass + + +class RunArgs: + def __init__( + self, waitURL="" + ): + self.waitURL = waitURL + + +class Bench: + def __init__(self, name, category="other"): + self.name = name + self.category = category + + def __str__(self): + return json.dumps(self.__dict__) + + def set_tag(self, tag): + self.name = f"{self.name}:{tag}" + + +class BenchRunner: + CMD_URL_WAIT = { + "wordpress": RunArgs(waitURL="http://localhost:80"), + } + + # complete listing + ALL = dict( + [ + ("wordpress", Bench("wordpress", "web-framework")), + ] + ) + + def __init__( + self, + registry="localhost:5000", + snapshotter="overlayfs", + cleanup=True, + insecure_registry=False, + ): + self.registry = registry + if self.registry != "": + self.registry += "/" + + self.snapshotter = snapshotter + self.insecure_registry = insecure_registry + + self.cleanup = cleanup + + def image_ref(self, repo): + return posixpath.join(self.registry, repo) + + def run(self, bench): + repo = image_repo(bench.name) + if repo in BenchRunner.CMD_URL_WAIT: + return self.run_cmd_url_wait( + repo=bench.name, runargs=BenchRunner.CMD_URL_WAIT[repo] + ) + else: + print("Unknown bench: " + repo) + sys.exit(1) + + def run_cmd_url_wait(self, repo, runargs): + image_ref = self.image_ref(repo) + container_id = repo.replace(":", "-") + + pull_cmd = self.pull_cmd(image_ref) + print(pull_cmd) + print("Pulling image %s ..." % image_ref) + with timer(pull_cmd) as t: + pull_elapsed = t + + create_cmd = self.create_cmd_url_wait_cmd(image_ref, container_id, runargs) + print(create_cmd) + + print("Creating container for image %s ..." % image_ref) + with timer(create_cmd) as t: + create_elapsed = t + + run_cmd = self.task_start_cmd(container_id, iteration=False) + print(run_cmd) + + print("Running container %s ..." % container_id) + start_run = datetime.now() + + _ = subprocess.Popen(run_cmd, shell=True) + while True: + try: + req = urllib.request.urlopen(runargs.waitURL) + print(req.status) + req.close() + break + except: + time.sleep(0.01) + + end_run = datetime.now() + run_elapsed = datetime.timestamp(end_run) - datetime.timestamp(start_run) + + print("Run time: %f s" % run_elapsed) + + if self.cleanup: + self.clean_up(image_ref, container_id) + + return pull_elapsed, create_elapsed, run_elapsed + + def pull_cmd(self, image_ref): + insecure_flag = "--insecure-registry" if self.insecure_registry else "" + return ( + f"sudo nerdctl --snapshotter {self.snapshotter} pull {insecure_flag} {image_ref}" + ) + + def create_cmd_url_wait_cmd(self, image_ref, container_id, runargs): + cmd = f"sudo nerdctl --snapshotter {self.snapshotter} create --net=host " + cmd += f"--name={container_id} {image_ref}" + return cmd + + def task_start_cmd(self, container_id, iteration: bool): + if iteration: + return f"sudo nerdctl --snapshotter {self.snapshotter} start -a {container_id}" + else: + return f"sudo nerdctl --snapshotter {self.snapshotter} start {container_id}" + + def task_kill_cmd(self, container_id): + return f"sudo nerdctl --snapshotter {self.snapshotter} stop {container_id}" + + def clean_up(self, image_ref, container_id): + print("Cleaning up environment for %s ..." % container_id) + cmd = self.task_kill_cmd(container_id) + print(cmd) + rc = os.system(cmd) # sometimes containers already exit. we ignore the failure. + cmd = f"sudo nerdctl --snapshotter {self.snapshotter} rm -f {container_id}" + print(cmd) + rc = os.system(cmd) + assert rc == 0 + cmd = f"sudo nerdctl --snapshotter {self.snapshotter} rmi -f {image_ref}" + print(cmd) + rc = os.system(cmd) + assert rc == 0 + + +def image_repo(ref: str): + return ref.split(":")[0] + + +def image_tag(ref: str) -> str: + try: + return ref.split(":")[1] + except IndexError: + return None + + +def bench_image(local_registry, insecure_local_registry, image, f: TextIOWrapper, snapshotter="overlayfs"): + try: + bench = copy.deepcopy(BenchRunner.ALL[image_repo(image)]) + tag = image_tag(image) + if tag is not None: + bench.set_tag(tag) + except KeyError: + logging.warning("image %s not supported, skip", image) + sys.exit(1) + runner = BenchRunner( + registry=local_registry, + snapshotter=snapshotter, + cleanup=True, + insecure_registry=insecure_local_registry, + ) + pull_elapsed, create_elapsed, run_elapsed = runner.run(bench) + total_elapsed = f"{pull_elapsed + create_elapsed + run_elapsed: .6f}" + pull_elapsed = f"{pull_elapsed: .6f}" + create_elapsed = f"{create_elapsed: .6f}" + run_elapsed = f"{run_elapsed: .6f}" + line = f"{bench.name},{pull_elapsed},{create_elapsed},{run_elapsed},{total_elapsed}" + f.writelines(line + "\n") + f.flush() diff --git a/misc/benchmark/benchmark.py b/misc/benchmark/benchmark.py new file mode 100644 index 00000000000..439810a703f --- /dev/null +++ b/misc/benchmark/benchmark.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +from argparse import ArgumentParser + +import bench as bench +import convert as cvt +import metrics +import prefetch_list as alg +import util +import yaml + +CONFIG = "config.yml" +PREFETCH_FILE_LIST = "out_list.txt" + + +def main(): + """ + 1. read config file to knows the images:tag and registry and if we need convert to nydus image or not + 2. we have four modes for benchmark, the first is oci, the second is nydus without prefetch, the third is nydus with prefetch all, the latest is nydus with prefetch file list + """ + parser = ArgumentParser() + parser.add_argument( + "--mode", + choices=["oci", "nydus-no-prefetch", "nydus-all-prefetch", "nydus-filelist-prefetch"], + dest="mode", + type=str, + required=True, + help="The mode of benchmark. Available modes are: oci, nydus-none_prefetch, nydus-all-prefetch, nydus-filelist-prefetch." + ) + args = parser.parse_args() + mode = args.mode + + # read config.yml + cfg = {} + with open(CONFIG, 'r', encoding='utf-8') as f: + try: + cfg = yaml.load(stream=f, Loader=yaml.FullLoader) + except Exception as inst: + print('error reading config file') + print(inst) + exit(-1) + # bench + start_bench(cfg, cfg["image"], mode) + util.show_csv(util.image_repo(cfg["image"]) + ".csv") + + +def collect_metrics(cfg: dict, image: str) -> str: + """ + collect metrics + """ + return metrics.collect(cfg["local_registry"], cfg["insecure_local_registry"], util.image_nydus(image)) + + +def start_bench(cfg: dict, image: str, mode: str): + """ + bench oci, nydus without prefetch, nydus with all prefetch, nydus witch prefetch file list + """ + f = open(util.image_repo(image) + ".csv", "w") + csv_headers = "repo,pull_elapsed(s),create_elapsed(s),run_elapsed(s),total_elapsed(s)" + f.writelines(csv_headers + "\n") + f.flush() + if mode == "oci": + util.enable_wondersphaper(cfg["bandwith"]) + bench.bench_image(cfg["local_registry"], cfg["insecure_local_registry"], image, f) + elif mode == "nydus-no-prefetch": + util.enable_wondersphaper(cfg["bandwith"]) + bench.bench_image(cfg["local_registry"], cfg["insecure_local_registry"], util.image_nydus(image), f, "nydus") + elif mode == "nydus-all-prefetch": + # open prefetch enable + util.switch_config_prefetch_enable() + util.reload_nydus() + util.enable_wondersphaper(cfg["bandwith"]) + bench.bench_image(cfg["local_registry"], cfg["insecure_local_registry"], util.image_nydus(image), f, "nydus") + else: + # opne the metrics colletc api + util.switch_config_access_pattern() + util.reload_nydus() + # collect metrics data + file = collect_metrics(cfg, image) + # generate prefetch list + _ = alg.get_prefetch_list(file) + # rebuild + cvt.convert_nydus_prefetch(cfg["source_registry"], cfg["insecure_source_registry"], cfg["local_registry"], cfg["insecure_local_registry"], image, PREFETCH_FILE_LIST) + # open prefetch enable + util.switch_config_prefetch_enable() + # close the metrics colletc api + util.switch_config_access_pattern() + util.reload_nydus() + util.enable_wondersphaper(cfg["bandwith"]) + bench.bench_image(cfg["local_registry"], cfg["insecure_local_registry"], util.image_nydus_prefetch(image), f, "nydus") + + +if __name__ == "__main__": + main() diff --git a/misc/benchmark/cni_bridge.conf b/misc/benchmark/cni_bridge.conf new file mode 100644 index 00000000000..faff51944af --- /dev/null +++ b/misc/benchmark/cni_bridge.conf @@ -0,0 +1,15 @@ +{ + "cniVersion": "0.4.0", + "name": "bridge", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "10.10.0.0/16", + "routes": [ + { "dst": "0.0.0.0/0" } + ] + } +} diff --git a/misc/benchmark/config.yml b/misc/benchmark/config.yml new file mode 100644 index 00000000000..404c1dc4dbd --- /dev/null +++ b/misc/benchmark/config.yml @@ -0,0 +1,6 @@ +source_registry: docker.io +insecure_source_registry: False +local_registry: localhost:5000 +insecure_local_registry: True +bandwith: 81920 +image: wordpress:latest diff --git a/misc/benchmark/containerd_config.toml b/misc/benchmark/containerd_config.toml new file mode 100644 index 00000000000..a17dadf9cd8 --- /dev/null +++ b/misc/benchmark/containerd_config.toml @@ -0,0 +1,9 @@ +[proxy_plugins] +[proxy_plugins.nydus] + type = "snapshot" + address = "/run/containerd-nydus/containerd-nydus-grpc.sock" + +[plugins.cri] +[plugins.cri.containerd] + snapshotter = "nydus" + disable_snapshot_annotations = false diff --git a/misc/benchmark/convert.py b/misc/benchmark/convert.py new file mode 100644 index 00000000000..c6a85e351d5 --- /dev/null +++ b/misc/benchmark/convert.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import os + + +class Image: + def __init__(self, source_registry, insecure_source_registry, target_registry, insecure_target_registry, image, prefetch=""): + """ + the prefetch is the file path of prefetch list file,and it is optional + """ + self.source_registry = source_registry + self.insecure_source_registry = insecure_source_registry + self.target_registry = target_registry + self.insecure_target_registry = insecure_target_registry + self.image = image + self.prefetch = prefetch + + def image_repo(self): + return self.image.split(":")[0] + + def image_tag(self) -> str: + try: + return self.image.split(":")[1] + except IndexError: + return None + + def convert_cmd(self): + if self.prefetch == "": + target_image = self.image_repo() + ":" + self.image_tag() + "_nydus" + if self.insecure_source_registry and self.insecure_target_registry: + return f"sudo nydusify convert --source {self.source_registry}/{self.image} --target {self.target_registry}/{target_image} --source-insecure --target-insecure" + elif self.insecure_source_registry: + return f"sudo nydusify convert --source {self.source_registry}/{self.image} --target {self.target_registry}/{target_image} --source-insecure" + elif self.insecure_target_registry: + return f"sudo nydusify convert --source {self.source_registry}/{self.image} --target {self.target_registry}/{target_image} --target-insecure" + else: + return f"sudo nydusify convert --source {self.source_registry}/{self.image} --target {self.target_registry}/{target_image}" + else: + target_image = self.image_repo() + ":" + self.image_tag() + "_nydus_prefetch" + if self.insecure_source_registry and self.insecure_target_registry: + return f"sudo cat {self.prefetch} | nydusify convert --prefetch-patterns --source {self.source_registry}/{self.image} --target {self.target_registry}/{target_image} --source-insecure --target-insecure" + elif self.insecure_source_registry: + return f"sudo cat {self.prefetch} | nydusify convert --prefetch-patterns --source {self.source_registry}/{self.image} --target {self.target_registry}/{target_image} --source-insecure" + elif self.insecure_target_registry: + return f"sudo cat {self.prefetch} | nydusify convert --prefetch-patterns --source {self.source_registry}/{self.image} --target {self.target_registry}/{target_image} --target-insecure" + else: + return f"sudo cat {self.prefetch} | nydusify convert --prefetch-patterns --source {self.source_registry}/{self.image} --target {self.target_registry}/{target_image}" + + def nydus_convert(self): + """ + convert oci image to nydus image (prefetchfile is optional) + """ + print(self.convert_cmd()) + rc = os.system(self.convert_cmd()) + assert rc == 0 + + +def convert_nydus_prefetch(source_registry: str, insecure_source_registry: bool, target_registry: str, insecure_target_registry: bool, image: str, prefetch: str): + """ + convert nydus with prefetch image api + """ + Image(source_registry, + insecure_source_registry, + target_registry, + insecure_target_registry, + image, prefetch).nydus_convert() diff --git a/misc/benchmark/metrics.py b/misc/benchmark/metrics.py new file mode 100755 index 00000000000..4f1b3c1fe4a --- /dev/null +++ b/misc/benchmark/metrics.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +import csv +import json +import os +import posixpath +import random +import re +import shutil +import string +import subprocess +import time +import urllib.request + +""" +define some file path, and sock api url +""" +LOG_DIR = "log" +TEMP_DIR = "tmp" +URL_PREFIX = "http://localhost/api" +ACCESS_PATTERN_METRICS = "/v1/metrics/pattern" +BOOTSTRAP_DIR = "/var/lib/containerd-nydus/snapshots" +API_DIR = "/var/lib/containerd-nydus/socket" + + +class MetricsCollector: + def __init__(self, cfg: dict): + self.registry = cfg["registry"] + self.insecure_registry = cfg["insecure_registry"] + self.image = cfg["image"] + + def start_collet(self, arg="") -> str: + image_ref = self.image_ref(self.image) + container_name = self.image.replace(":", "-") + random_string() + pull_cmd = self.pull_cmd(image_ref) + print(pull_cmd) + print("Pulling image %s ..." % image_ref) + rc = os.system(pull_cmd) + assert rc == 0 + create_cmd = self.create_container_cmd(image_ref, container_name, arg) + print(create_cmd) + print("Creating container for image %s ..." % image_ref) + rc = os.system(create_cmd) + assert rc == 0 + run_cmd = self.start_container_cmd(container_name) + print(run_cmd) + print("Running container %s ..." % container_name) + rc = os.system(run_cmd) + assert rc == 0 + file = self.collect(self.image) + self.clean_up(image_ref, container_name) + return file + + def image_ref(self, repo): + return posixpath.join(self.registry, repo) + + def pull_cmd(self, image_ref): + insecure_flag = "--insecure-registry" if self.insecure_registry else "" + return ( + f"sudo nerdctl --snapshotter nydus pull {insecure_flag} {image_ref}" + ) + + def create_container_cmd(self, image_ref, container_id, arg=""): + if arg == "": + return f"sudo nerdctl --snapshotter nydus create --net=host --name={container_id} {image_ref}" + else: + return f"sudo nerdctl --snapshotter nydus create --net=host {arg} --name={container_id} {image_ref}" + + def start_container_cmd(self, container_id): + return f"sudo nerdctl --snapshotter nydus start {container_id}" + + def stop_container_cmd(self, container_id): + return f"sudo nerdctl --snapshotter nydus stop {container_id}" + + def clean_up(self, image_ref, container_id): + print("Cleaning up environment for %s ..." % container_id) + cmd = self.stop_container_cmd(container_id) + print(cmd) + rc = os.system(cmd) + assert rc == 0 + cmd = f"sudo nerdctl --snapshotter nydus rm -f {container_id}" + print(cmd) + rc = os.system(cmd) + assert rc == 0 + cmd = f"sudo nerdctl --snapshotter nydus rmi -f {image_ref}" + print(cmd) + rc = os.system(cmd) + assert rc == 0 + + def collect(self, repo) -> str: + """ + TODO: update to the condition, here is just for wait the wordpress up + """ + # wait wordpress response in 80 port + while True: + try: + req = urllib.request.urlopen("http://localhost:80") + print(req.status) + req.close() + break + except: + time.sleep(0.01) + socket = search_file(API_DIR, "api.sock") + if socket == None: + print("can't find the api.sock") + exit(1) + bootstrap = search_file(BOOTSTRAP_DIR, "image.boot") + if bootstrap == None: + print("can't find the bootstrap") + exit(1) + + # bootstrap + bootstap_data = check_bootstrap(bootstrap) + + # access_pattern + access_pattern = get_access_pattern(socket, bootstap_data) + + header = ["file_path", "first_access_time", "file_size"] + + file_name = posixpath.join(repo + ".csv") + if not os.path.exists(file_name): + os.mknod(file_name) + with open(file_name, 'w', newline='') as f: + writer = csv.writer(f) + writer.writerow(header) + for item in access_pattern: + writer.writerow( + [item.file_path, item.first_access_time_secs * 10**9 + item.first_access_time_nanos, item.file_size]) + return file_name + + +def get_file_by_bootstrap(bootstrap, inode): + """ + load the data of bootstrap + """ + with open(bootstrap, 'r') as file: + for line in file: + if line.startswith("inode:"): + result = re.search(r'"([^"]+)".*ino (\d+).*i_size (\d+)', line) + value_file_path = result.group(1) + value_ino = result.group(2) + value_file_size = result.group(3) + if int(value_ino) == inode: + return value_file_path, value_file_size + return None, None + + +def check_bootstrap(bootstrap): + """ + use nydus-image to get the data of bootstap + """ + file_path = random_string() + cmd = ["sudo", "nydus-image", "check"] + cmd.extend(["-B", bootstrap, "-v"]) + with open(TEMP_DIR + "/" + file_path, 'w') as f: + _ = run_cmd( + cmd, + shell=True, + stdout=f, + stderr=f, + ) + return TEMP_DIR + "/" + file_path + + +class AccessPattern: + def __init__(self, file_path, first_access_time_secs, first_access_time_nanos, file_size): + self.file_path = file_path + self.first_access_time_secs = first_access_time_secs + self.first_access_time_nanos = first_access_time_nanos + self.file_size = file_size + + +def get_access_pattern(sock, bootstap_data): + """ + get the file access pattern from the sock + """ + contents = "" + # The api occasionally returns incomplete information + while contents.endswith("]") == False: + with open(send_request(sock, ACCESS_PATTERN_METRICS), 'r') as file: + contents = file.read() + resp = json.loads(contents) + access_pattern_list = [] + for item in resp: + item['first_access_time_secs'] = item['first_access_time_secs'] + item["first_access_time_nanos"] = item["first_access_time_nanos"] + file_path, file_size = get_file_by_bootstrap(bootstap_data, item['ino']) + access_pattern_list.append(AccessPattern( + file_path, item['first_access_time_secs'], item['first_access_time_nanos'], file_size)) + return access_pattern_list + + +def random_string(): + """ + generate a random string of fixed length + """ + return "".join(random.choice(string.ascii_lowercase) for _ in range(10)) + + +def search_file(root_dir, file_name): + """ + search the bootsatrap and api.scok of the image, but only return the first match file, + so we need to clear the images and containers befor we start metrics.py + """ + for subdir, _, files in os.walk(root_dir): + if file_name in files: + return os.path.join(subdir, file_name) + return None + + +def send_request(sock_path, url): + """ + send request to the local socket with the url. + save the response in the file. + """ + file_path = random_string() + cmd = ["sudo", "curl", "--unix-socket", sock_path] + cmd.extend(["-X", "GET", URL_PREFIX + url]) + # caruon + with open(TEMP_DIR + "/" + file_path, 'w') as f: + _ = run_cmd( + cmd, + shell=True, + stdout=f, + stderr=subprocess.PIPE + ) + return TEMP_DIR + "/" + file_path + + +def run_cmd(cmd, wait: bool = True, **kwargs): + """ + run a cmd with the subprocess + """ + shell = kwargs.pop("shell", False) + if shell: + cmd = " ".join(cmd) + popen_obj = subprocess.Popen(cmd, shell=shell, **kwargs) + if wait: + popen_obj.wait() + return popen_obj.returncode, popen_obj + + +def init(): + if os.path.exists(TEMP_DIR): + if os.path.isdir(TEMP_DIR): + shutil.rmtree(TEMP_DIR) + else: + os.remove(TEMP_DIR) + os.mkdir(TEMP_DIR) + + +def collect(local_registry, insecure_local_registry, image) -> str: + """ + serve for benchmark.py + """ + init() + cfg = {"registry": local_registry, "insecure_registry": insecure_local_registry, "image": image} + file = MetricsCollector(cfg).start_collet() + shutil.rmtree(TEMP_DIR) + return file diff --git a/misc/benchmark/nydus-snapshotter.service b/misc/benchmark/nydus-snapshotter.service new file mode 100644 index 00000000000..ba35f7ef31b --- /dev/null +++ b/misc/benchmark/nydus-snapshotter.service @@ -0,0 +1,18 @@ +[Unit] +Description=nydus snapshotter +After=network.target +Before=containerd.service + +[Service] +Type=simple +Environment=HOME=/root +ExecStart=/usr/local/bin/containerd-nydus-grpc --config-path /etc/nydus/config.json +Restart=always +RestartSec=1 +KillMode=process +OOMScoreAdjust=-999 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/misc/benchmark/nydusd_config.json b/misc/benchmark/nydusd_config.json new file mode 100644 index 00000000000..8e5cfe957bc --- /dev/null +++ b/misc/benchmark/nydusd_config.json @@ -0,0 +1,30 @@ +{ + "device": { + "backend": { + "type": "registry", + "config": { + "scheme": "http", + "host": "localhost:5000" + } + }, + "cache": { + "type": "blobcache", + "config": { + "work_dir": "/var/lib/nydus/cache/" + } + } + }, + "mode": "direct", + "digest_validate": false, + "enable_xattr": true, + "iostats_files": false, + "access_pattern": false, + "latest_read_files": false, + "fs_prefetch": { + "enable": false, + "threads_count": 10, + "merging_size": 1048576, + "bandwidth_rate": 10485760, + "prefetch_all": false + } +} diff --git a/misc/benchmark/prefetch_list.py b/misc/benchmark/prefetch_list.py new file mode 100755 index 00000000000..4ce8aaddd1e --- /dev/null +++ b/misc/benchmark/prefetch_list.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import csv +from functools import cmp_to_key +from typing import Tuple + + +def read_csv(csv_path: str) -> dict: + """ + convert csv to dictionary + """ + file_dict = {} + with open(csv_path, 'r') as fp: + reader = csv.DictReader(fp) + for x in reader: + temp = {} + temp['file_size'] = str(x['file_size']) + temp['first_access_time'] = str(x['first_access_time']) + file_dict[x['file_path']] = temp + return file_dict + + +def sort_list(csv_path: str) -> Tuple[dict, list]: + """ + sort access_files + """ + file_dict = read_csv(csv_path) + file_list = list(file_dict.keys()) + + def comp(a, b): + at = file_dict[a]['first_access_time'] + bt = file_dict[b]['first_access_time'] + if len(at) > len(bt): + return 1 + elif len(at) < len(bt): + return -1 + else: + for i in range(len(at)): + if i == len(at)-1: + return -1 + if int(at[i]) > int(bt[i]): + return 1 + elif int(at[i]) < int(bt[i]): + return -1 + else: + continue + + file_list.sort(key=cmp_to_key(comp)) + + return file_dict, file_list + + +def optimize_list(csv_path: str) -> list: + """ + optimize sorted list - only sort + """ + _, file_list = sort_list(csv_path) + + return file_list + + +def to_txt(file_list: list, outpath: str): + """ + prefetch_list to txt + """ + with open(outpath, 'w')as f: + for k in file_list: + f.write(k + '\n') + + +def get_prefetch_list(csv_path: str) -> list: + """ + get prefetch_list + """ + optimized_list = optimize_list(csv_path) + to_txt(optimized_list, 'out_list.txt') + return optimized_list diff --git a/misc/benchmark/util.py b/misc/benchmark/util.py new file mode 100644 index 00000000000..09ced9cec56 --- /dev/null +++ b/misc/benchmark/util.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +import csv +import json +import os + +NYDUSD_CONFIG = "/etc/nydus/config.json" + + +def get_nydusd_config() -> dict: + config = [] + with open(NYDUSD_CONFIG, 'r', encoding='utf-8') as f: + config = json.load(f) + return config + + +def switch_config_prefetch_enable(): + """ + switch the fs_prefetch.enable status + """ + config = get_nydusd_config() + config["fs_prefetch"]["enable"] = not config["fs_prefetch"]["enable"] + with open(NYDUSD_CONFIG, 'w', encoding='utf-8') as f: + json.dump(config, f, ensure_ascii=False, indent=4) + + +def switch_config_access_pattern(): + """ + switch the status of access pattern used by metrics.py + """ + config = get_nydusd_config() + config["iostats_files"] = not config["iostats_files"] + config["access_pattern"] = not config["access_pattern"] + config["latest_read_files"] = not config["latest_read_files"] + with open(NYDUSD_CONFIG, 'w', encoding='utf-8') as f: + json.dump(config, f, ensure_ascii=False, indent=4) + + +def reload_nydus(): + rc = os.system("systemctl restart nydus-snapshotter.service") + assert rc == 0 + + +def image_repo(ref: str): + return ref.split(":")[0] + + +def image_tag(ref: str) -> str: + try: + return ref.split(":")[1] + except IndexError: + return None + + +def image_nydus(ref: str): + return image_repo(ref) + ":" + image_tag(ref) + "_nydus" + + +def image_nydus_prefetch(ref: str) -> str: + return image_repo(ref) + ":" + image_tag(ref) + "_nydus_prefetch" + + +def show_csv(file_path: str): + with open(file_path, mode='r') as f: + reader = csv.reader(f) + for row in reader: + formatted_row = [f'{cell}' for cell in row] + print(','.join(formatted_row)) + + +def enable_wondersphaper(bandwith: int): + os.system("sudo wondershaper -a docker0 -u " + str(bandwith) + " -d" + str(bandwith))