Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Don't assume runners have more than 2 cores available for benchmarking #5827

Merged
merged 1 commit into from
Feb 22, 2022
Merged

Don't assume runners have more than 2 cores available for benchmarking #5827

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 24 additions & 24 deletions tests/benchmark/test_baseline_benchmarks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,37 +142,37 @@ def test_baseline_benchmark_j1(self, benchmark):
linter.msg_status == 0
), f"Expected no errors to be thrown: {pprint.pformat(linter.reporter.messages)}"

def test_baseline_benchmark_j10(self, benchmark):
def test_baseline_benchmark_j2(self, benchmark):
"""Establish a baseline of pylint performance with no work across threads.

Same as `test_baseline_benchmark_j1` but we use -j10 with 10 fake files to
Same as `test_baseline_benchmark_j1` but we use -j2 with 2 fake files to
ensure end-to-end-system invoked.

Because this is also so simple, if this regresses something very serious has
happened.
"""
linter = PyLinter(reporter=Reporter())
linter.config.jobs = 10
linter.config.jobs = 2

# Create file per worker, using all workers
fileinfos = [self.empty_filepath for _ in range(linter.config.jobs)]

assert linter.config.jobs == 10
assert linter.config.jobs == 2
assert len(linter._checkers) == 1, "Should have 'master'"
benchmark(linter.check, fileinfos)
assert (
linter.msg_status == 0
), f"Expected no errors to be thrown: {pprint.pformat(linter.reporter.messages)}"

def test_baseline_benchmark_check_parallel_j10(self, benchmark):
"""Should demonstrate times very close to `test_baseline_benchmark_j10`."""
def test_baseline_benchmark_check_parallel_j2(self, benchmark):
"""Should demonstrate times very close to `test_baseline_benchmark_j2`."""
linter = PyLinter(reporter=Reporter())

# Create file per worker, using all workers
fileinfos = [self.empty_file_info for _ in range(linter.config.jobs)]

assert len(linter._checkers) == 1, "Should have 'master'"
benchmark(check_parallel, linter, jobs=10, files=fileinfos)
benchmark(check_parallel, linter, jobs=2, files=fileinfos)
assert (
linter.msg_status == 0
), f"Expected no errors to be thrown: {pprint.pformat(linter.reporter.messages)}"
Expand All @@ -196,20 +196,20 @@ def test_baseline_lots_of_files_j1(self, benchmark):
linter.msg_status == 0
), f"Expected no errors to be thrown: {pprint.pformat(linter.reporter.messages)}"

def test_baseline_lots_of_files_j10(self, benchmark):
"""Establish a baseline with only 'master' checker being run in -j10.
def test_baseline_lots_of_files_j2(self, benchmark):
"""Establish a baseline with only 'master' checker being run in -j2.

As with the -j1 variant above `test_baseline_lots_of_files_j1`, we do not
register any checkers except the default 'master', so the cost is just that of
the check_parallel system across 10 workers, plus the overhead of PyLinter
the check_parallel system across 2 workers, plus the overhead of PyLinter
"""
if benchmark.disabled:
benchmark(print, "skipping, only benchmark large file counts")
return # _only_ run this test is profiling
linter = PyLinter(reporter=Reporter())
linter.config.jobs = 10
linter.config.jobs = 2
fileinfos = [self.empty_filepath for _ in range(self.lot_of_files)]
assert linter.config.jobs == 10
assert linter.config.jobs == 2
assert len(linter._checkers) == 1, "Should have 'master'"
benchmark(linter.check, fileinfos)
assert (
Expand All @@ -236,8 +236,8 @@ def test_baseline_lots_of_files_j1_empty_checker(self, benchmark):
linter.msg_status == 0
), f"Expected no errors to be thrown: {pprint.pformat(linter.reporter.messages)}"

def test_baseline_lots_of_files_j10_empty_checker(self, benchmark):
"""Baselines pylint for a single extra checker being run in -j10, for N-files.
def test_baseline_lots_of_files_j2_empty_checker(self, benchmark):
"""Baselines pylint for a single extra checker being run in -j2, for N-files.

We use a checker that does no work, so the cost is just that of the system at
scale, across workers
Expand All @@ -246,10 +246,10 @@ def test_baseline_lots_of_files_j10_empty_checker(self, benchmark):
benchmark(print, "skipping, only benchmark large file counts")
return # _only_ run this test is profiling
linter = PyLinter(reporter=Reporter())
linter.config.jobs = 10
linter.config.jobs = 2
linter.register_checker(NoWorkChecker(linter))
fileinfos = [self.empty_filepath for _ in range(self.lot_of_files)]
assert linter.config.jobs == 10
assert linter.config.jobs == 2
assert len(linter._checkers) == 2, "Should have 'master' and 'sleeper'"
benchmark(linter.check, fileinfos)
assert (
Expand All @@ -260,7 +260,7 @@ def test_baseline_benchmark_j1_single_working_checker(self, benchmark):
"""Establish a baseline of single-worker performance for PyLinter.

Here we mimic a single Checker that does some work so that we can see the
impact of running a simple system with -j1 against the same system with -j10.
impact of running a simple system with -j1 against the same system with -j2.

We expect this benchmark to take very close to
`numfiles*SleepingChecker.sleep_duration`
Expand All @@ -272,8 +272,8 @@ def test_baseline_benchmark_j1_single_working_checker(self, benchmark):
linter.register_checker(SleepingChecker(linter))

# Check the same number of files as
# `test_baseline_benchmark_j10_single_working_checker`
fileinfos = [self.empty_filepath for _ in range(10)]
# `test_baseline_benchmark_j2_single_working_checker`
fileinfos = [self.empty_filepath for _ in range(2)]

assert linter.config.jobs == 1
assert len(linter._checkers) == 2, "Should have 'master' and 'sleeper'"
Expand All @@ -282,27 +282,27 @@ def test_baseline_benchmark_j1_single_working_checker(self, benchmark):
linter.msg_status == 0
), f"Expected no errors to be thrown: {pprint.pformat(linter.reporter.messages)}"

def test_baseline_benchmark_j10_single_working_checker(self, benchmark):
def test_baseline_benchmark_j2_single_working_checker(self, benchmark):
"""Establishes baseline of multi-worker performance for PyLinter/check_parallel.

We expect this benchmark to take less time that test_baseline_benchmark_j1,
`error_margin*(1/J)*(numfiles*SleepingChecker.sleep_duration)`

Because of the cost of the framework and system the performance difference will
*not* be 1/10 of -j1 versions.
*not* be 1/2 of -j1 versions.
"""
if benchmark.disabled:
benchmark(print, "skipping, do not want to sleep in main tests")
return # _only_ run this test is profiling
linter = PyLinter(reporter=Reporter())
linter.config.jobs = 10
linter.config.jobs = 2
linter.register_checker(SleepingChecker(linter))

# Check the same number of files as
# `test_baseline_benchmark_j1_single_working_checker`
fileinfos = [self.empty_filepath for _ in range(10)]
fileinfos = [self.empty_filepath for _ in range(2)]

assert linter.config.jobs == 10
assert linter.config.jobs == 2
assert len(linter._checkers) == 2, "Should have 'master' and 'sleeper'"
benchmark(linter.check, fileinfos)
assert (
Expand Down