Skip to content

Commit

Permalink
Remove blocking calls and change threat intel feed flow to event driv…
Browse files Browse the repository at this point in the history
…en (opensearch-project#871)

* remove actionGet() and change threat intel feed flow to event driven

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* fix javadocs

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* revert try catch removals

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* use action listener wrap() in detector threat intel code paths

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* add try catch

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

---------

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
  • Loading branch information
eirsep authored Mar 2, 2024
1 parent 8ef0a3f commit 172d58d
Show file tree
Hide file tree
Showing 10 changed files with 265 additions and 370 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import static org.opensearch.securityanalytics.model.Detector.DETECTORS_INDEX;
Expand Down Expand Up @@ -121,35 +119,24 @@ public void createDocLevelQueryFromThreatIntel(List<LogType.IocFields> iocFieldL
listener.onResponse(Collections.emptyList());
return;
}

CountDownLatch latch = new CountDownLatch(1);
threatIntelFeedDataService.getThreatIntelFeedData(new ActionListener<>() {
@Override
public void onResponse(List<ThreatIntelFeedData> threatIntelFeedData) {
if (threatIntelFeedData.isEmpty()) {
listener.onResponse(Collections.emptyList());
} else {
listener.onResponse(
createDocLevelQueriesFromThreatIntelList(iocFieldList, threatIntelFeedData, detector)
);
threatIntelFeedDataService.getThreatIntelFeedData(ActionListener.wrap(
threatIntelFeedData -> {
if (threatIntelFeedData.isEmpty()) {
listener.onResponse(Collections.emptyList());
} else {
listener.onResponse(
createDocLevelQueriesFromThreatIntelList(iocFieldList, threatIntelFeedData, detector)
);
}
}, e -> {
log.error("Failed to get threat intel feeds for doc level query creation", e);
listener.onFailure(e);
}
latch.countDown();
}

@Override
public void onFailure(Exception e) {
log.error("Failed to get threat intel feeds for doc level query creation", e);
listener.onFailure(e);
latch.countDown();
}
});

latch.await(30, TimeUnit.SECONDS);
} catch (InterruptedException e) {
log.error("Failed to create doc level queries from threat intel feeds", e);
));
} catch (Exception e) {
log.error("Failed to create doc level query from threat intel data", e);
listener.onFailure(e);
}

}

private static String constructId(Detector detector, String iocType) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,12 @@
import org.opensearch.core.xcontent.ToXContent;
import org.opensearch.core.xcontent.XContentBuilder;
import org.opensearch.securityanalytics.model.ThreatIntelFeedData;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction;
import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest;
import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse;
import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata;
import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext;
import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings;
import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata;
import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService;
import org.opensearch.securityanalytics.util.IndexUtils;
import org.opensearch.securityanalytics.util.SecurityAnalyticsException;
Expand All @@ -56,7 +56,6 @@
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.CountDownLatch;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -104,21 +103,13 @@ public void getThreatIntelFeedData(
ActionListener<List<ThreatIntelFeedData>> listener
) {
try {

String tifdIndex = getLatestIndexByCreationDate();
if (tifdIndex == null) {
createThreatIntelFeedData(listener);
} else {
SearchRequest searchRequest = new SearchRequest(tifdIndex);
searchRequest.source().size(9999); //TODO: convert to scroll
String finalTifdIndex = tifdIndex;
client.search(searchRequest, ActionListener.wrap(r -> listener.onResponse(ThreatIntelFeedDataUtils.getTifdList(r, xContentRegistry)), e -> {
log.error(String.format(
"Failed to fetch threat intel feed data from system index %s", finalTifdIndex), e);
listener.onFailure(e);
}));
fetchThreatIntelFeedDataFromIndex(tifdIndex, listener);
}
} catch (InterruptedException e) {
} catch (Exception e) {
log.error("Failed to get threat intel feed data", e);
listener.onFailure(e);
}
Expand Down Expand Up @@ -150,21 +141,16 @@ public void createIndexIfNotExists(final String indexName, final ActionListener<
.mapping(getIndexMapping()).timeout(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT));
StashedThreadContext.run(
client,
() -> client.admin().indices().create(createIndexRequest, new ActionListener<>() {
@Override
public void onResponse(CreateIndexResponse response) {
if (response.isAcknowledged()) {
listener.onResponse(response);
} else {
onFailure(new OpenSearchStatusException("Threat intel feed index creation failed", RestStatus.INTERNAL_SERVER_ERROR));
}
}

@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
})
() -> client.admin().indices().create(createIndexRequest,
ActionListener.wrap(
response -> {
if (response.isAcknowledged())
listener.onResponse(response);
else
listener.onFailure(new OpenSearchStatusException("Threat intel feed index creation failed", RestStatus.INTERNAL_SERVER_ERROR));

}, listener::onFailure
))
);
}

Expand Down Expand Up @@ -223,28 +209,20 @@ public void parseAndSaveThreatIntelFeedDataCSV(
}
bulkRequestList.add(bulkRequest);

GroupedActionListener<BulkResponse> bulkResponseListener = new GroupedActionListener<>(new ActionListener<>() {
@Override
public void onResponse(Collection<BulkResponse> bulkResponses) {
int idx = 0;
for (BulkResponse response: bulkResponses) {
BulkRequest request = bulkRequestList.get(idx);
if (response.hasFailures()) {
throw new OpenSearchException(
"error occurred while ingesting threat intel feed data in {} with an error {}",
StringUtils.join(request.getIndices()),
response.buildFailureMessage()
);
}
GroupedActionListener<BulkResponse> bulkResponseListener = new GroupedActionListener<>(ActionListener.wrap(bulkResponses -> {
int idx = 0;
for (BulkResponse response : bulkResponses) {
BulkRequest request = bulkRequestList.get(idx);
if (response.hasFailures()) {
throw new OpenSearchException(
"error occurred while ingesting threat intel feed data in {} with an error {}",
StringUtils.join(request.getIndices()),
response.buildFailureMessage()
);
}
listener.onResponse(new ThreatIntelIndicesResponse(true, List.of(indexName)));
}

@Override
public void onFailure(Exception e) {
listener.onFailure(e);
}
}, bulkRequestList.size());
listener.onResponse(new ThreatIntelIndicesResponse(true, List.of(indexName)));
}, listener::onFailure), bulkRequestList.size());

for (int i = 0; i < bulkRequestList.size(); ++i) {
saveTifds(bulkRequestList.get(i), timeout, bulkResponseListener);
Expand Down Expand Up @@ -291,52 +269,47 @@ public void deleteThreatIntelDataIndex(final List<String> indices) {
.prepareDelete(indices.toArray(new String[0]))
.setIndicesOptions(IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN)
.setTimeout(clusterSettings.get(SecurityAnalyticsSettings.THREAT_INTEL_TIMEOUT))
.execute(new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse response) {
if (response.isAcknowledged() == false) {
onFailure(new OpenSearchException("failed to delete data[{}]", String.join(",", indices)));
}
}

@Override
public void onFailure(Exception e) {
log.error("unknown exception:", e);
}
})
.execute(ActionListener.wrap(
response -> {
if (response.isAcknowledged() == false) {
log.error(new OpenSearchException("failed to delete threat intel feed index[{}]",
String.join(",", indices)));
}
}, e -> log.error("failed to delete threat intel feed index [{}]", e)
))
);
}

private void createThreatIntelFeedData(ActionListener<List<ThreatIntelFeedData>> listener) throws InterruptedException {
CountDownLatch countDownLatch = new CountDownLatch(1);
private void createThreatIntelFeedData(ActionListener<List<ThreatIntelFeedData>> listener) {
client.execute(
PutTIFJobAction.INSTANCE,
new PutTIFJobRequest("feed_updater", clusterSettings.get(SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL)),
new ActionListener<>() {
@Override
public void onResponse(AcknowledgedResponse acknowledgedResponse) {
log.debug("Acknowledged threat intel feed updater job created");
countDownLatch.countDown();
String tifdIndex = getLatestIndexByCreationDate();

SearchRequest searchRequest = new SearchRequest(tifdIndex);
searchRequest.source().size(9999); //TODO: convert to scroll
String finalTifdIndex = tifdIndex;
client.search(searchRequest, ActionListener.wrap(r -> listener.onResponse(ThreatIntelFeedDataUtils.getTifdList(r, xContentRegistry)), e -> {
log.error(String.format(
"Failed to fetch threat intel feed data from system index %s", finalTifdIndex), e);
ActionListener.wrap(
r -> {
if (false == r.isAcknowledged()) {
listener.onFailure(new Exception("Failed to acknowledge Put Tif job action"));
return;
}
log.debug("Acknowledged threat intel feed updater job created");
String tifdIndex = getLatestIndexByCreationDate();
fetchThreatIntelFeedDataFromIndex(tifdIndex, listener);
}, e -> {
log.debug("Failed to create threat intel feed updater job", e);
listener.onFailure(e);
}));
}

@Override
public void onFailure(Exception e) {
log.debug("Failed to create threat intel feed updater job", e);
countDownLatch.countDown();
}
}
}
)
);
countDownLatch.await();
}

private void fetchThreatIntelFeedDataFromIndex(String tifdIndex, ActionListener<List<ThreatIntelFeedData>> listener) {
SearchRequest searchRequest = new SearchRequest(tifdIndex);
searchRequest.source().size(9999); //TODO: convert to scroll
String finalTifdIndex = tifdIndex;
client.search(searchRequest, ActionListener.wrap(r -> listener.onResponse(ThreatIntelFeedDataUtils.getTifdList(r, xContentRegistry)), e -> {
log.error(String.format(
"Failed to fetch threat intel feed data from system index %s", finalTifdIndex), e);
listener.onFailure(e);
}));
}

private String getIndexMapping() {
Expand Down
Loading

0 comments on commit 172d58d

Please sign in to comment.