Skip to content

Commit

Permalink
Update benchmarks with Python
Browse files Browse the repository at this point in the history
  • Loading branch information
@henry2004y
henry2004y committed Dec 16, 2022
1 parent 54678ae commit b756e11
Show file tree
Hide file tree
Showing 3 changed files with 190 additions and 139 deletions.
118 changes: 67 additions & 51 deletions benchmark/perf.jl
View file
Original file line number Diff line number Diff line change
@@ -1,66 +1,82 @@
# Script for generating benchmark results in
# https://henry2004y.github.io/Vlasiator.jl/dev/benchmark/
#
# Note: this script is not intended for execution as a whole: each benchmark needs to be
# executed independently after importing all the required packages.

using Vlasiator
using Vlasiator: RE
using Vlasiator, PyPlot
using BenchmarkTools
using Glob

## Reading DCCRG variables
ENV["MPLBACKEND"]="agg" # no GUI for Matplotlib

file1 = "bulk.singleprecision.vlsv" # 80 B
file2 = "bulk.0000003.vlsv" # 900 KB
file3 = "bulk1.0001000.vlsv" # 32 MB
# Select file
file = file1
# Load metadata
@time meta = load(file);
@time meta = load(file);
@time meta = load(file);
# Select variable name
var = "proton/vg_rho";
# Read variable, sorted
@time rho = meta[var];
@time rho = meta[var];
# Read variable, unsorted
@time rho_unsorted = readvariable($meta, $var, false);
@time rho_unsorted = readvariable($meta, $var, false);
files = ["1d_single.vlsv", "bulk.2d.vlsv", "2d_double.vlsv", "2d_AFC.vlsv", "3d_EGI.vlsv"]
versions = [5,5,5,4,5]

## Plotting with PyPlot
# Log color scale is used for comparison with Analysator.

# 2D density contour on a uniform mesh
filename = "bulk.0000501.vlsv"
meta = load(filename)
@time pcolormesh(meta, "rho", colorscale=Log)
close()
@time pcolormesh(meta, "rho", colorscale=Log)
# Download test files if not found in the current path
for i in eachindex(files)
if isfile(files[i])
@info "Benchmark file $(files[i]) found..."
continue
elseif i == 4
url_base = "https://a3s.fi/swift/v1/AUTH_81f1cd490d494224880ea77e4f98490d/vlasiator-2d-afc/"
filename = "production_halfres/bulk.0000000.vlsv"
file_origin = basename(filename)
url = joinpath(url_base, filename)
@info "Downloading 1.8G test file..."
run(`curl -o $file_new $url`)
run(`mv $(file_origin) $file`)
elseif i in (1,2,3)
@info "Downloading test files..."
run(`curl -o testdata.tar.gz https://raw.githubusercontent.com/henry2004y/vlsv_data/master/testdata.tar.gz`)
run(`curl -o 1d_single.vlsv https://raw.githubusercontent.com/henry2004y/vlsv_data/master/1d_single.vlsv`)
run(`curl -o 2d_double.vlsv https://raw.githubusercontent.com/henry2004y/vlsv_data/master/2d_double.vlsv`)
run(`tar -xzvf testdata.tar.gz`)
elseif i == 5
@warn "$(files[i]) is not open-access!"
end
end

# 2D density slices from 3D AMR mesh
filename = "bulk1.0001000.vlsv"
meta = load(filename)
@time pcolormesh(meta, "proton/vg_rho", colorscale=Log)
close()
@time pcolormesh(meta, "proton/vg_rho", colorscale=Log)
# Define a parent BenchmarkGroup to contain our suite
const suite = BenchmarkGroup()

## Virtual satellite tracking at a static location
# Add children groups and tags to our benchmark suite.
suite["load"] = BenchmarkGroup(files)
suite["read"] = BenchmarkGroup(files)
suite["plot"] = BenchmarkGroup(["2d", "3d"])

# 3D EGI data on Turso, University of Helsinki
dir = "/wrk-vakka/group/spacephysics/vlasiator/3D/EGI/bulk/dense_cold_hall1e5_afterRestart374/"
# Add benchmarks
for (i, file) in enumerate(files)
suite["load"][file] = @benchmarkable load($file)
if i == 4
var = "proton/rho"
suite["read"][file*"_sorted"] = @benchmarkable meta[$var] setup=(meta=load($file))
suite["read"][file*"_unsorted"] =
@benchmarkable readvariable(meta, $var, false) setup=(meta=load($file))
suite["plot"]["contour_uniform"] =
@benchmarkable pcolormesh(meta, $var, colorscale=Log) setup=(meta=load($file))
else
var = "proton/vg_rho"
suite["read"][file*"_sorted"] = @benchmarkable meta[$var] setup=(meta=load($file))
suite["read"][file*"_unsorted"] =
@benchmarkable readvariable(meta, $var, false) setup=(meta=load($file))
end
if i == 5
var = "proton/vg_rho"
suite["plot"]["contour_nonuniform"] =
@benchmarkable pcolormesh(meta, $var, colorscale=Log) setup=(meta=load($file))
end

filenames = glob("bulk1*.vlsv", dir)
end

var = "proton/vg_rho"
loc = [12, 0, 0] .* RE
# If a cache of tuned parameters already exists, use it, otherwise, tune and cache
# the benchmark parameters. Reusing cached parameters is faster and more reliable
# than re-tuning `suite` every time the file is included.
paramspath = joinpath(dirname(@__FILE__), "params.json")

println("Number of files: ", length(filenames))
# Static cell ID
id = let
meta = load(filenames[1])
getcell(meta, loc)
if isfile(paramspath)
loadparams!(suite, BenchmarkTools.load(paramspath)[1], :evals)
else
tune!(suite)
BenchmarkTools.save(paramspath, params(suite))
end

@btime data_series = extractsat($filenames, $var, $id)
results = run(suite, verbose=true, samples=100, seconds=20)

show(results)
143 changes: 77 additions & 66 deletions benchmark/perf.py
View file
Original file line number Diff line number Diff line change
@@ -1,83 +1,94 @@
# Script for generating benchmark results in
# https://henry2004y.github.io/Vlasiator.jl/dev/benchmark/
#
# Note: this script is not intended for execution as a whole: each benchmark
# needs to be executed independently after importing all the required packages.

import timeit
import requests, tarfile, timeit, os, os.path, textwrap
import pytools as pt
import matplotlib
matplotlib.use('agg')

os.environ["PTNONINTERACTIVE"] = "1"

files = ['1d_single.vlsv', 'bulk.2d.vlsv', '2d_double.vlsv', '2d_AFC.vlsv', '3d_EGI.vlsv']

for i, file in enumerate(files):
if os.path.isfile(file):
print(f"Benchmark file {file} found...")
elif i == 3:
url_base = 'https://a3s.fi/swift/v1/AUTH_81f1cd490d494224880ea77e4f98490d/vlasiator-2d-afc/'
filename = 'production_halfres/bulk.0000000.vlsv'
r = requests.get(url_base+filename, allow_redirects=True)
open('2d_AFC.vlsv', 'wb').write(r.content)
elif i in [0,1,2]:
r = requests.get('https://raw.githubusercontent.com/henry2004y/vlsv_data/master/testdata.tar.gz', allow_redirects=True)
open('testdata.tar.gz', 'wb').write(r.content)
r = requests.get('https://raw.githubusercontent.com/henry2004y/vlsv_data/master/1d_single.vlsv', allow_redirects=True)
open('1d_single.vlsv', 'wb').write(r.content)
r = requests.get('https://raw.githubusercontent.com/henry2004y/vlsv_data/master/2d_double.vlsv', allow_redirects=True)
open('2d_double.vlsv', 'wb').write(r.content)

file = tarfile.open('testdata.tar.gz')
file.extractall('./')
file.close()
elif i == 4:
print(f"{file} is not open-access!")

# Number of trails for each benchmark
ntrail = 100

# Setup: importing packages
p0 = """
import pytools as pt
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
matplotlib.use('agg')
"""
# Setup: selecting file and variable
p1 = p0+"""
file1 = 'bulk.singleprecision.vlsv' # 80 B
file2 = 'bulk.0000003.vlsv' # 900 KB
file3 = 'bulk1.0001000.vlsv' # 32 MB

file = file1
var = 'proton/vg_rho'
"""
# Setup: reading metadata
p2 = p0+p1+"""
f = pt.vlsvfile.VlsvReader(file)
"""
# Setup: virtual satellite extraction
p3 = p0+"""
import glob
dir = "/wrk-vakka/group/spacephysics/vlasiator/3D/EGI/bulk/dense_cold_hall1e5_afterRestart374/"
var = "proton/vg_rho"
Re = 6.371e6 # Earth radius, [m]
loc = [12*Re, 0, 0]
filenames = sorted(glob.glob(dir+"bulk1*.vlsv"))
data_series = np.zeros(len(filenames))
f = pt.vlsvfile.VlsvReader(filenames[0])
id = f.get_cellid(loc)
print(id)
"""
for i, file in enumerate(files):
# Setup: reading metadata
p2 = textwrap.dedent("""\
f = pt.vlsvfile.VlsvReader(file)
cellID = f.read_variable('CellID')
cell_sorted = np.argsort(cellID)
""")

if i != 3:
# Setup: selecting file and variable
p1 = p0 + textwrap.dedent(f"""\
file = '{file}'
var = 'proton/vg_rho'
""")
else:
# Setup: selecting file and variable
p1 = p0 + textwrap.dedent(f"""\
file = '{file}'
var = 'proton/rho'
""")

# Run: reading metadata
s1 = p2
t1 = timeit.timeit(stmt=s1, setup=p1, number=ntrail) / ntrail * 1e3
print(f"{file}:")
print(f"Reading metadata in {t1:0.4f} ms")
# Run: obtaining sorted scalar variable
s2 = textwrap.dedent("""
rho = f.read_variable(var)[cell_sorted]
""")
t2 = timeit.timeit(stmt=s2, setup=p0+p1+p2, number=ntrail) / ntrail * 1e3
print(f"{file}:")
print(f"Reading scalar DCCRG variable in {t2:0.4f} ms")

# Run: reading metadata
s1 = """
f = pt.vlsvfile.VlsvReader(file)
"""
# Run: obtaining scalar variable
s2 = """
cellID = f.read_variable('CellID')
cell_sorted = np.argsort(cellID)
rho = f.read_variable(var)[cell_sorted]
"""
# Run: plotting density contour on a uniform mesh
s3 = """
pt.plot.plot_colormap(filename='bulk.0000501.vlsv', var='rho', draw=1)
s3 = f"""
pt.plot.plot_colormap(filename='{files[3]}', var='rho', draw=1)
"""
t3 = timeit.timeit(stmt=s3, setup=p0, number=5) / 5
print(f"Uniform 2d plotting in {t3:0.4f} s")

# Run: plotting density slice from a 3D AMR mesh
s4 = """
pt.plot.plot_colormap3dslice(filename='bulk1.0001000.vlsv', var='proton/vg_rho', draw=1, normal='y')
"""
# Run: virtual satellite extraction from a static location
s5 = """
for (i,fname) in enumerate(filenames):
f = pt.vlsvfile.VlsvReader(fname)
data_series[i] = f.read_variable(name=var, cellids=id)
s4 = f"""
pt.plot.plot_colormap3dslice(filename='{files[4]}', var='proton/vg_rho', draw=1, normal='y')
"""

print(timeit.timeit(stmt=s1, setup=p1, number=10) / 10 * 1e3) # [ms]
print(timeit.timeit(stmt=s2, setup=p2, number=10) / 10 * 1e3) # [ms]

t1 = timeit.timeit(stmt=s1, setup=p1, number=10) / 10
print(f"Finished reading metadata in {t1:0.4f} ms")

t2 = timeit.timeit(stmt=s2, setup=p2, number=10) / 10
print(f"Finished reading scalar DCCRG variable in {t2:0.4f} ms")

t3 = timeit.timeit(stmt=s3, setup=p0, number=3) / 3
print(f"Finished plotting in {t3:0.4f} s")

t4 = timeit.timeit(stmt=s4, setup=p0, number=3) / 3
print(f"Finished plotting in {t4:0.4f} s")

t5 = timeit.timeit(stmt=s5, setup=p3, number=1) / 1
print(f"Finished virtual satellite tracking in {t5:0.4f} s")
t4 = timeit.timeit(stmt=s4, setup=p0, number=5) / 5
print(f"AMR slice plotting in {t4:0.4f} s")
68 changes: 46 additions & 22 deletions docs/src/benchmark.md
View file
Original file line number Diff line number Diff line change
@@ -1,41 +1,65 @@
# Benchmarks

The test file information are listed below:

| Index | Filename | Number of Cells | Dimension | AMR | Public |
|:-------|:----------------|:---------------:|:---------:|:---:|:------:|
| 1 | 1d_single.vlsv | 20 | 1 | No | Yes |
| 2 | bulk.2d.vlsv | 6,300 | 1 | No | Yes |
| 3 | 2d_double.vlsv | 51,200 | 2 | No | Yes |
| 4 | 2d_AFC.vlsv | 4,612,500 | 2 | No | Yes |
| 5 | 3d_EGI.vlsv | 3,966,580 | 3 | Yes | No |

Access to the public data can be found from [vlsv_data](https://github.com/henry2004y/vlsv_data).

!!! note
The numbers shown here are comparisons between Analysator v0.9 and Vlasiator.jl v0.9.32 running Python 3.6.9/3.9.7 and Julia 1.8.3 with the scripts [`perf.jl`](https://github.com/henry2004y/Vlasiator.jl/blob/master/benchmark/perf.jl) and [`perf.py`](https://github.com/henry2004y/Vlasiator.jl/blob/master/benchmark/perf.py). The timings are collected from a i5-10210U @ 1.6GHz CPU with 16 GB RAM if not specified.

* Loading meta data[^1]
| File Index | Julia [ms] | Python [ms] | Ratio |
|:-----------|:----------:|:-----------:|:-----:|
| 1 | 0.18 | 1.19 | 6.6 |
| 2 | 0.51 | 1.66 | 3.1 |
| 3 | 2.33 | 3.11 | 1.3 |
| 4 | 506 | 277 | 0.5 |
| 5 | 549 | 283 | 0.5 |

[^1]: See the [issue](https://github.com/henry2004y/Vlasiator.jl/issues/124) about sorting. The performance of EzXML is also not ideal: we may need to find a better XML parser in Julia.

* Reading DCCRG grid variables
| Size | Julia [ms] | Python [ms] | Speedup |
|:----------------|:----------:|:-----------:|:-------:|
| 80 B Float32 | 0.002 | 0.14 | 55 |
| 25 KiB Float64 | 0.017 | 0.44 | 26 |
| 879 KiB Float64 | 0.3 | 8.7 | 29 |
| 30 MiB Float64 | 10.8 | 295 | 27 |
| File Index | Julia [ms] | Python [ms] | Ratio |
|:-----------|:----------:|:-----------:|:------:|
| 1 | 0.004 | 0.07 | 17 |
| 2 | 0.02[^2] | 0.08 | 4 |
| 3 | 0.17[^2] | 0.21 | 1.2 |
| 4 | 20[^2] | 23 | 1.1 |
| 5 | 11[^2] | 11 | 1.0 |

[^1]: Vlasiator.jl can be even faster if there is no conversion from Float64 to Float32. See [Precision](log.md#precision).
[^2]: Vlasiator.jl can be faster if there is no conversion from Float64 to Float32. See [Precision](log.md#precision).

* Reading field solver grid variables[^2]
| Size | Julia [s] | Python [s] | Speedup |
|:----------------|:---------:|:----------:|:-------:|
| 6.2 GiB Float32 | 8 | 61 | 7.6 |
* Reading field solver grid variables[^3]
| File Index | Size | Julia [s] | Python [s] | Ratio |
|:-----------|:----------------|:---------:|:----------:|:-----:|
| 5 | 6.2 GiB Float32 | 8 | 61 | 7.6 |

[^2]: The field solver grid is a regular Cartesian grid at the finest refinement level. Therefore the storage requirement for fsgrid variables are quite significant: with limited memory (e.g. 16 GB RAM) you may encounter out-of-memory issues when reading `fg_b` more than once. In Vlasiator.jl, we provide the option `usemmap=true` for reading large arrays --- see [Memory](log.md#memory) for more.
[^3]: The field solver grid is a regular Cartesian grid at the finest refinement level. Therefore the storage requirement for fsgrid variables are quite significant: with limited memory (e.g. 16 GB RAM) you may encounter out-of-memory issues when reading `fg_b` more than once. In Vlasiator.jl, we provide the option `usemmap=true` for reading large arrays --- see [Memory](log.md#memory) for more.

* Plotting 2D density contours on a uniform mesh[^3]
| Size | Julia, 1st time [s] | Julia, 2nd time [s] | Python [s] | Speedup |
|:---------|:-------------------:|:-------------------:|:----------:|:-------:|
| 26.7 MiB | 2.4 | 0.5 | 4.7 | 9.4 |
* Plotting 2D density contours on a uniform mesh
| File Index | Julia [s] | Python [s] | Ratio |
|:-----------|:---------:|:----------:|:-----:|
| 4 | 0.5 | 12.3 | 9.4 |

* Plotting 2D density slices from a 3D AMR mesh
| Size | Julia, 1st time [s] | Julia, 2nd time [s] | Python [s] | Speedup |
|:---------|:-------------------:|:-------------------:|:----------:|:-------:|
| 30.5 MiB | 2.7 | 0.5 | 5.0 | 10 |
| Size | Julia [s][^4] | Python [s] | Ratio |
|:---------|:-------------:|:----------:|:-----:|
| 30.5 MiB | 0.5 | 4.6 | 10 |

[^3]: The inefficieny of JIT for the first time execution is a famous problem in the Julia community known as "Time-To-First-X".
[^4]: The first time execution will be slower due to JIT compilation (which is excluded in the timing here). This is known as "Time-To-First-X" in the Julia community.

* Static virtual satellite tracking from 3D AMR data (26G per frame, 32 MB Cell IDs) on a cluster[^4]
* Static virtual satellite tracking from 3D AMR data (26G per frame, 32 MB Cell IDs) on a cluster[^5]

| Frames | Julia, 1 thread [s] | Julia, 2 threads [s] | Python [s] |
|:-------|:-------------------:|:--------------------:|:----------:|
| 845 | 45 | 34 | 1220 |

[^4]: University of Helsinki cluster Vorna with Intel Xeon E5-2697 @ 2.70GHz.
[^5]: University of Helsinki cluster Vorna with Intel Xeon E5-2697 @ 2.70GHz.

0 comments on commit b756e11

Please sign in to comment.