Skip to content

Commit

Permalink
Adding option to skip aggregate output timing metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
dlarge committed Jan 28, 2025
1 parent f9e58fa commit 82d6526
Show file tree
Hide file tree
Showing 4 changed files with 16 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -215,9 +215,11 @@ extensions, storageConfig, dataInput, verifyFlag, batchSize, loadStepId()
Loggers.MSG.debug("{}: item output file aggregator initialized", loadStepId());
}

itemTimingMetricsOutputFileAggregators.add(
if (!config.boolVal("output-metrics-skip-timing")) {
itemTimingMetricsOutputFileAggregators.add(
new ItemTimingMetricOutputFileAggregator(loadStepId(), fileMgrs));
Loggers.MSG.debug("{}: item metrics output file aggregator initialized", loadStepId());
Loggers.MSG.debug("{}: item metrics output file aggregator initialized", loadStepId());
}

if(config.boolVal("output-metrics-trace-persist")) {
opTraceLogFileAggregators.add(new OpTraceLogFileAggregator(loadStepId(), fileMgrs));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import java.nio.file.FileSystemException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -70,16 +71,17 @@ public TimingMetricQuantileResultsImpl(final List<Double> quantileValues, final
})
.collect(Collectors.toList());
if (tmpMetrics.size() != nodeAmount) {
metricsValues = new HashMap<>(); // No metrics
Loggers.ERR.warn(
"Expected to aggregate timing metrics from {} node(s), but found {} local file(s) after aggregation for pattern {}",
nodeAmount, tmpMetrics.size(), metricsFilePattern);
} else {
// merge sorted metrics arrays from each aggregated local tmp file and take the quantile values from it
// we do not want to store the whole values list in the class as we can eventually reach a few Gb size array of
// latency/duration values.
// So we only store a few values of specified quantiles
metricsValues = retrieveQuantileValues(quantileValues, mergeSort(tmpMetrics));
}

// merge sorted metrics arrays from each aggregated local tmp file and take the quantile values from it
// we do not want to store the whole values list in the class as we can eventually reach a few Gb size array of
// latency/duration values.
// So we only store a few values of specified quantiles
metricsValues = retrieveQuantileValues(quantileValues, mergeSort(tmpMetrics));
}

private File[] findAllWorkersMetricsFiles() {
Expand Down
2 changes: 2 additions & 0 deletions src/main/resources/config-schema.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ output:
persist: boolean
threshold: double
quantiles: list
skip:
timing: boolean
run:
id: long
comment: string
Expand Down
2 changes: 2 additions & 0 deletions src/main/resources/config/defaults.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,8 @@ output:
- 0.25
- 0.5
- 0.75
skip:
timing: false

run:
id: 0
Expand Down

0 comments on commit 82d6526

Please sign in to comment.