Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

action: convert ci can show the metrics collected during conversion #1279

Merged
merged 2 commits into from
May 17, 2023
Merged

action: convert ci can show the metrics collected during conversion #1279

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dfc054a
fix: nydusify save metric to specify the file path
Desiki-high May 17, 2023
404941d
action: convert ci show metrics during conversion
Desiki-high May 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 74 additions & 3 deletions .github/workflows/convert.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ jobs:
sudo chmod +x /usr/local/bin/nydus*
sudo chmod +x /usr/local/bin/fsck.erofs
sudo docker run -d --restart=always -p 5000:5000 registry
sudo mkdir convert-zran
for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
echo "converting $I:latest to $I:nydus-nightly-oci-ref"
ghcr_repo=${{ env.REGISTRY }}/${{ env.ORGANIZATION }}
Expand All @@ -137,7 +138,8 @@ jobs:
--oci-ref \
--source localhost:5000/$I \
--target localhost:5000/$I:nydus-nightly-oci-ref \
--platform linux/amd64,linux/arm64
--platform linux/amd64,linux/arm64 \
--output-json convert-zran/${I}.json

# check zran image and referenced oci image
sudo rm -rf ./tmp
Expand All @@ -148,6 +150,11 @@ jobs:
sudo fsck.erofs -d1 output/nydus_bootstrap
sudo rm -rf ./output
done
- name: Save Nydusify Metric
uses: actions/upload-artifact@v3
with:
name: convert-zran-metric
path: convert-zran

convert-native-v5:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -175,6 +182,7 @@ jobs:
run: |
sudo chmod +x /usr/local/bin/nydus*
sudo docker run -d --restart=always -p 5000:5000 registry
sudo mkdir convert-native-v5
for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
echo "converting $I:latest to $I:nydus-nightly-v5"
# for pre-built images
Expand All @@ -183,17 +191,24 @@ jobs:
--target ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/$I:nydus-nightly-v5 \
--fs-version 5 \
--platform linux/amd64,linux/arm64

# use local registry for speed
sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
--source $I:latest \
--target localhost:5000/$I:nydus-nightly-v5 \
--fs-version 5 \
--platform linux/amd64,linux/arm64
--platform linux/amd64,linux/arm64 \
--output-json convert-native-v5/${I}.json

sudo rm -rf ./tmp
sudo DOCKER_CONFIG=$HOME/.docker nydusify check --source $I:latest \
--target localhost:5000/$I:nydus-nightly-v5
done
- name: Save Nydusify Metric
uses: actions/upload-artifact@v3
with:
name: convert-native-v5-metric
path: convert-native-v5

convert-native-v6:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -227,6 +242,7 @@ jobs:
sudo chmod +x /usr/local/bin/nydus*
sudo chmod +x /usr/local/bin/fsck.erofs
sudo docker run -d --restart=always -p 5000:5000 registry
sudo mkdir convert-native-v6
for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
echo "converting $I:latest to $I:nydus-nightly-v6"
# for pre-built images
Expand All @@ -235,12 +251,14 @@ jobs:
--target ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/$I:nydus-nightly-v6 \
--fs-version 6 \
--platform linux/amd64,linux/arm64

# use local registry for speed
sudo DOCKER_CONFIG=$HOME/.docker nydusify convert \
--source $I:latest \
--target localhost:5000/$I:nydus-nightly-v6 \
--fs-version 6 \
--platform linux/amd64,linux/arm64
--platform linux/amd64,linux/arm64 \
--output-json convert-native-v6/${I}.json

sudo rm -rf ./tmp
sudo DOCKER_CONFIG=$HOME/.docker nydusify check --source $I:latest \
Expand All @@ -249,3 +267,56 @@ jobs:
sudo fsck.erofs -d1 output/nydus_bootstrap
sudo rm -rf ./output
done
- name: Save Nydusify Metric
uses: actions/upload-artifact@v3
with:
name: convert-native-v6-metric
path: convert-native-v6

convert-metric:
runs-on: ubuntu-latest
needs: [convert-zran, convert-native-v5, convert-native-v6]
steps:
- name: Checkout repository
uses: actions/checkout@v3
- name: Download Zran Metric
uses: actions/download-artifact@master
with:
name: convert-zran-metric
path: convert-zran
- name: Download V5 Metric
uses: actions/download-artifact@master
with:
name: convert-native-v5-metric
path: convert-native-v5
- name: Download V6 Metric
uses: actions/download-artifact@master
with:
name: convert-native-v6-metric
path: convert-native-v6
- name: Summary
run: |
echo "## Image Size(MB)" > $GITHUB_STEP_SUMMARY
echo "|image name|nydus-zran|nydus-v5|nydus-v6|" >> $GITHUB_STEP_SUMMARY
echo "|:---:|:--:|:-------:|:-------:|" >> $GITHUB_STEP_SUMMARY
for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
zranSourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-zran/${I}.json) / 1048576")")
zranTargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-zran/${I}.json) / 1048576")")
v5SourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-native-v5/${I}.json) / 1048576")")
v5TargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-native-v5/${I}.json) / 1048576")")
v6SourceImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.SourceImageSize' convert-native-v6/${I}.json) / 1048576")")
v6TargetImageSize=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.TargetImageSize' convert-native-v6/${I}.json) / 1048576")")
echo "|${I}|${zranSourceImageSize}/${zranTargetImageSize}|${v5SourceImageSize}/${v5TargetImageSize}|${v6SourceImageSize}/${v6TargetImageSize}|" >> $GITHUB_STEP_SUMMARY
done
echo "## Conversion Time(ms)" >> $GITHUB_STEP_SUMMARY
echo "|image name|nydus-zran|nydus-v5|nydus-v6|" >> $GITHUB_STEP_SUMMARY
echo "|:---:|:--:|:-------:|:-------:|" >> $GITHUB_STEP_SUMMARY
for I in $(cat ${{ env.IMAGE_LIST_PATH }}); do
zranConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-zran/${I}.json) / 1000000")")
v5ConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v5/${I}.json) / 1000000")")
v6ConversionElapsed=$(printf "%0.2f" "$(bc <<< "scale=2; $(jq -r '.ConversionElapsed' convert-native-v6/${I}.json) / 1000000")")
echo "|${I}|${zranConversionElapsed}|${v5ConversionElapsed}|${v6ConversionElapsed}|" >> $GITHUB_STEP_SUMMARY
done
- uses: geekyeggo/delete-artifact@v2
with:
name: '*'
8 changes: 4 additions & 4 deletions contrib/nydusify/cmd/nydusify.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -409,10 +409,10 @@ func main() {
Usage: "Path to the nydus-image binary, default to search in PATH",
EnvVars: []string{"NYDUS_IMAGE"},
},
&cli.BoolFlag{
&cli.StringFlag{
Name: "output-json",
Value: false,
Usage: "Enable saving the metrics collected during conversion in JSON file output.json",
Value: "",
Usage: "File path to save the metrics collected during conversion in JSON format, for example: './output.json'",
EnvVars: []string{"OUTPUT_JSON"},
},
},
Expand Down Expand Up @@ -510,7 +510,7 @@ func main() {
AllPlatforms: c.Bool("all-platforms"),
Platforms: c.String("platform"),

OutputJSON: c.Bool("output-json"),
OutputJSON: c.String("output-json"),
}

return converter.Convert(context.Background(), opt)
Expand Down
6 changes: 3 additions & 3 deletions contrib/nydusify/pkg/converter/converter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ type Opt struct {
AllPlatforms bool
Platforms string

OutputJSON bool
OutputJSON string
}

func Convert(ctx context.Context, opt Opt) error {
Expand Down Expand Up @@ -90,8 +90,8 @@ func Convert(ctx context.Context, opt Opt) error {
}

metric, err := cvt.Convert(ctx, opt.Source, opt.Target)
if opt.OutputJSON {
dumpMetric(metric, opt.WorkDir)
if opt.OutputJSON != "" {
dumpMetric(metric, opt.OutputJSON)
}
return err
}
5 changes: 2 additions & 3 deletions contrib/nydusify/pkg/converter/metric.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,13 @@ package converter
import (
"encoding/json"
"os"
"path/filepath"

"github.com/goharbor/acceleration-service/pkg/converter"
"github.com/pkg/errors"
)

func dumpMetric(metric *converter.Metric, workDir string) error {
file, err := os.Create(filepath.Join(workDir, "output.json"))
func dumpMetric(metric *converter.Metric, path string) error {
file, err := os.Create(path)
if err != nil {
return errors.Wrap(err, "Create file for metric")
}
Expand Down