Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Turn off Statsbeat when proxy is used or any exception from the server #2221

Merged
merged 17 commits into from
Apr 13, 2022
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,15 @@ private static AppIdSupplier start(Instrumentation instrumentation) {
RpConfigurationPolling.startPolling(rpConfiguration, config, telemetryClient, appIdSupplier);
}

// initialize StatsbeatModule
statsbeatModule.start(telemetryClient, config);
// initialize StatsbeatModule and don't start Statsbeat when proxy is used
String configProxyHost = config.proxy.host;
String systemPropertyProxyHost = System.getProperty("https.proxyHost");
if (configProxyHost == null
|| configProxyHost.isEmpty()
|| systemPropertyProxyHost == null
|| systemPropertyProxyHost.isEmpty()) {
statsbeatModule.start(telemetryClient, config);
}
trask marked this conversation as resolved.
Show resolved Hide resolved

// start local File purger scheduler task
if (!readOnlyFileSystem) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,14 +49,13 @@
import java.io.IOException;
import java.io.StringWriter;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.nio.channels.UnresolvedAddressException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.zip.GZIPOutputStream;
import javax.annotation.Nullable;
Expand Down Expand Up @@ -86,6 +85,9 @@ public class TelemetryChannel {
// operationLogger?
private final AtomicBoolean friendlyExceptionThrown = new AtomicBoolean();

private static final int MAX_STATSBEAT_ERROR_COUNT = 3;
private static final AtomicInteger statsbeatErrorCount = new AtomicInteger();

@SuppressWarnings("CatchAndPrintStackTrace")
private static ObjectMapper createObjectMapper() {
ObjectMapper mapper = new ObjectMapper();
Expand Down Expand Up @@ -342,7 +344,9 @@ private Consumer<HttpResponse> responseHandler(
}
},
exception -> {
operationLogger.recordFailure("exception retrieving response body", exception);
if (!isStatsbeat) {
operationLogger.recordFailure("exception retrieving response body", exception);
}
onFailure.accept(false);
});
}
Expand All @@ -364,19 +368,23 @@ private Consumer<Throwable> errorHandler(
String instrumentationKey, Consumer<Boolean> onFailure, OperationLogger operationLogger) {

return error -> {
if (isStatsbeat
&& (error instanceof UnknownHostException
|| error instanceof UnresolvedAddressException)) {
// when sending a Statsbeat request and server returns an UnknownHostException, it's
// likely that it's using AMPLS. In that case, we use the kill-switch to turn off Statsbeat.
if (isStatsbeat && statsbeatErrorCount.getAndIncrement() >= MAX_STATSBEAT_ERROR_COUNT) {
// when sending a Statsbeat request and server returns an Exception 3 times in a row, it's
// likely that it's using AMPLS or other private endpoints. In that case, we use the
// kill-switch to turn off Statsbeat.
// TODO need to figure out a way to detect AMPL or we can let the new ingestion service to
// handle this case for us when it becomes available.
statsbeatErrorCount.set(0);
trask marked this conversation as resolved.
Show resolved Hide resolved
heyams marked this conversation as resolved.
Show resolved Hide resolved
statsbeatModule.shutdown();
onFailure.accept(false);
return;
}

// TODO (trask) only log one-time friendly exception if no prior successes
if (!NetworkFriendlyExceptions.logSpecialOneTimeFriendlyException(
error, endpointUrl.toString(), friendlyExceptionThrown, logger)) {
// stop logging statsbeat failures
if (!isStatsbeat
&& !NetworkFriendlyExceptions.logSpecialOneTimeFriendlyException(
error, endpointUrl.toString(), friendlyExceptionThrown, logger)) {
operationLogger.recordFailure(
"Error sending telemetry items: " + error.getMessage(), error);
}
Expand Down