-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Integrate IO Usage Tracker to the Resource Usage Collector Service an…
…d Emit IO Usage Stats (#11880) * Integrate IO Usage Tracker to the Resource Usage Collector Service and Emit IO Usage Stats Signed-off-by: Ajay Kumar Movva <movvaam@amazon.com> * Enable IO Usage Tracker For Linux OS Signed-off-by: Ajay Kumar Movva <movvaam@amazon.com> * Addressing Comments Signed-off-by: Ajay Kumar Movva <movvaam@amazon.com> --------- Signed-off-by: Ajay Kumar Movva <movvaam@amazon.com> Co-authored-by: Ajay Kumar Movva <movvaam@amazon.com>
- Loading branch information
Showing
17 changed files
with
424 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
69 changes: 69 additions & 0 deletions
69
server/src/main/java/org/opensearch/node/IoUsageStats.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.node; | ||
|
|
||
| import org.opensearch.core.common.io.stream.StreamInput; | ||
| import org.opensearch.core.common.io.stream.StreamOutput; | ||
| import org.opensearch.core.common.io.stream.Writeable; | ||
| import org.opensearch.core.xcontent.ToXContentFragment; | ||
| import org.opensearch.core.xcontent.XContentBuilder; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.Locale; | ||
|
|
||
| /** | ||
| * This class is to store tne IO Usage Stats and used to return in node stats API. | ||
| */ | ||
| public class IoUsageStats implements Writeable, ToXContentFragment { | ||
|
|
||
| private double ioUtilisationPercent; | ||
|
|
||
| public IoUsageStats(double ioUtilisationPercent) { | ||
| this.ioUtilisationPercent = ioUtilisationPercent; | ||
| } | ||
|
|
||
| /** | ||
| * | ||
| * @param in the stream to read from | ||
| * @throws IOException if an error occurs while reading from the StreamOutput | ||
| */ | ||
| public IoUsageStats(StreamInput in) throws IOException { | ||
| this.ioUtilisationPercent = in.readDouble(); | ||
| } | ||
|
|
||
| /** | ||
| * Write this into the {@linkplain StreamOutput}. | ||
| * | ||
| * @param out the output stream to write entity content to | ||
| */ | ||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeDouble(this.ioUtilisationPercent); | ||
| } | ||
|
|
||
| public double getIoUtilisationPercent() { | ||
| return ioUtilisationPercent; | ||
| } | ||
|
|
||
| public void setIoUtilisationPercent(double ioUtilisationPercent) { | ||
| this.ioUtilisationPercent = ioUtilisationPercent; | ||
| } | ||
|
|
||
| @Override | ||
| public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { | ||
| builder.startObject(); | ||
| builder.field("max_io_utilization_percent", String.format(Locale.ROOT, "%.1f", this.ioUtilisationPercent)); | ||
| return builder.endObject(); | ||
| } | ||
|
|
||
| @Override | ||
| public String toString() { | ||
| return "IO utilization percent: " + String.format(Locale.ROOT, "%.1f", this.ioUtilisationPercent); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
102 changes: 102 additions & 0 deletions
102
server/src/main/java/org/opensearch/node/resource/tracker/AverageIoUsageTracker.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.node.resource.tracker; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.lucene.util.Constants; | ||
| import org.opensearch.common.ValidationException; | ||
| import org.opensearch.common.unit.TimeValue; | ||
| import org.opensearch.monitor.fs.FsInfo.DeviceStats; | ||
| import org.opensearch.monitor.fs.FsService; | ||
| import org.opensearch.node.IoUsageStats; | ||
| import org.opensearch.threadpool.ThreadPool; | ||
|
|
||
| import java.util.HashMap; | ||
| import java.util.Optional; | ||
|
|
||
| /** | ||
| * AverageIoUsageTracker tracks the IO usage by polling the FS Stats for IO metrics every (pollingInterval) | ||
| * and keeping track of the rolling average over a defined time window (windowDuration). | ||
| */ | ||
| public class AverageIoUsageTracker extends AbstractAverageUsageTracker { | ||
|
|
||
| private static final Logger LOGGER = LogManager.getLogger(AverageIoUsageTracker.class); | ||
| private final FsService fsService; | ||
| private final HashMap<String, Long> prevIoTimeDeviceMap; | ||
| private long prevTimeInMillis; | ||
| private IoUsageStats ioUsageStats; | ||
|
|
||
| public AverageIoUsageTracker(FsService fsService, ThreadPool threadPool, TimeValue pollingInterval, TimeValue windowDuration) { | ||
| super(threadPool, pollingInterval, windowDuration); | ||
| this.fsService = fsService; | ||
| this.prevIoTimeDeviceMap = new HashMap<>(); | ||
| this.prevTimeInMillis = -1; | ||
| this.ioUsageStats = null; | ||
| } | ||
|
|
||
| /** | ||
| * Get current IO usage percentage calculated using fs stats | ||
| */ | ||
| @Override | ||
| public long getUsage() { | ||
| long usage = 0; | ||
| Optional<ValidationException> validationException = this.preValidateFsStats(); | ||
| if (validationException != null && validationException.isPresent()) { | ||
| throw validationException.get(); | ||
| } | ||
| // Currently even during the raid setup we have only one mount device and it is giving 0 io time from /proc/diskstats | ||
| DeviceStats[] devicesStats = fsService.stats().getIoStats().getDevicesStats(); | ||
| long latestTimeInMillis = fsService.stats().getTimestamp(); | ||
| for (DeviceStats devicesStat : devicesStats) { | ||
| long devicePreviousIoTime = prevIoTimeDeviceMap.getOrDefault(devicesStat.getDeviceName(), (long) -1); | ||
| long deviceCurrentIoTime = devicesStat.ioTimeInMillis(); | ||
| if (prevTimeInMillis > 0 && (latestTimeInMillis - this.prevTimeInMillis > 0) && devicePreviousIoTime > 0) { | ||
| long absIoTime = (deviceCurrentIoTime - devicePreviousIoTime); | ||
| long deviceCurrentIoUsage = absIoTime * 100 / (latestTimeInMillis - this.prevTimeInMillis); | ||
| // We are returning the maximum IO Usage for all the attached devices | ||
| usage = Math.max(usage, deviceCurrentIoUsage); | ||
| } | ||
| prevIoTimeDeviceMap.put(devicesStat.getDeviceName(), devicesStat.ioTimeInMillis()); | ||
| } | ||
| this.prevTimeInMillis = latestTimeInMillis; | ||
| return usage; | ||
| } | ||
|
|
||
| @Override | ||
| protected void doStart() { | ||
| if (Constants.LINUX) { | ||
| this.ioUsageStats = new IoUsageStats(-1); | ||
| scheduledFuture = threadPool.scheduleWithFixedDelay(() -> { | ||
| long usage = getUsage(); | ||
| recordUsage(usage); | ||
| updateIoUsageStats(); | ||
| }, pollingInterval, ThreadPool.Names.GENERIC); | ||
| } | ||
| } | ||
|
|
||
| public Optional<ValidationException> preValidateFsStats() { | ||
| ValidationException validationException = new ValidationException(); | ||
| if (fsService == null | ||
| || fsService.stats() == null | ||
| || fsService.stats().getIoStats() == null | ||
| || fsService.stats().getIoStats().getDevicesStats() == null) { | ||
| validationException.addValidationError("FSService IoStats Or DeviceStats are Missing"); | ||
| } | ||
| return validationException.validationErrors().isEmpty() ? Optional.empty() : Optional.of(validationException); | ||
| } | ||
|
|
||
| private void updateIoUsageStats() { | ||
| this.ioUsageStats.setIoUtilisationPercent(this.isReady() ? this.getAverage() : -1); | ||
| } | ||
|
|
||
| public IoUsageStats getIoUsageStats() { | ||
| return this.ioUsageStats; | ||
| } | ||
| } |
Oops, something went wrong.