Skip to content

Commit

Permalink
add rollover & archival mechanism for correlation history indices
Browse files Browse the repository at this point in the history 
Signed-off-by: Subhobrata Dey <sbcd90@gmail.com>
  • Loading branch information
@sbcd90
sbcd90 committed Oct 17, 2023
1 parent 294785f commit 259808f
Show file tree
Hide file tree
Showing 19 changed files with 978 additions and 229 deletions.
4 changes: 4 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,10 @@ public List<Setting<?>> getSettings() {
SecurityAnalyticsSettings.FINDING_HISTORY_INDEX_MAX_AGE,
SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.CORRELATION_HISTORY_MAX_DOCS,
SecurityAnalyticsSettings.CORRELATION_HISTORY_INDEX_MAX_AGE,
SecurityAnalyticsSettings.CORRELATION_HISTORY_ROLLOVER_PERIOD,
SecurityAnalyticsSettings.CORRELATION_HISTORY_RETENTION_PERIOD,
SecurityAnalyticsSettings.IS_CORRELATION_INDEX_SETTING,
SecurityAnalyticsSettings.CORRELATION_TIME_WINDOW,
SecurityAnalyticsSettings.DEFAULT_MAPPING_SCHEMA,
Expand Down
26 changes: 13 additions & 13 deletions src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ public void insertCorrelatedFindings(String detectorType, Finding finding, Strin
searchSourceBuilder.fetchSource(true);
searchSourceBuilder.size(1);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(CorrelationIndices.CORRELATION_INDEX);
searchRequest.indices(CorrelationIndices.CORRELATION_METADATA_INDEX);

Check warning on line 76 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L76 

Added line #L76 was not covered by tests
searchRequest.source(searchSourceBuilder);
searchRequest.preference(Preference.PRIMARY_FIRST.type());

Expand Down Expand Up @@ -103,7 +103,7 @@ public void onResponse(SearchResponse response) {
searchSourceBuilder.fetchSource(true);
searchSourceBuilder.size(10000);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(CorrelationIndices.CORRELATION_INDEX);
searchRequest.indices(CorrelationIndices.CORRELATION_HISTORY_INDEX_PATTERN_REGEXP);

Check warning on line 106 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L106 

Added line #L106 was not covered by tests
searchRequest.source(searchSourceBuilder);
searchRequest.preference(Preference.PRIMARY_FIRST.type());

Expand Down Expand Up @@ -156,7 +156,7 @@ public void onResponse(MultiSearchResponse items) {
builder.field("scoreTimestamp", 0L);
builder.endObject();

IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_INDEX)
IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_HISTORY_WRITE_INDEX)

Check warning on line 159 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L159 

Added line #L159 was not covered by tests
.source(builder)
.timeout(indexTimeout);
bulkRequest.add(indexRequest);
Expand All @@ -183,7 +183,7 @@ public void onResponse(MultiSearchResponse items) {
corrBuilder.field("corrRules", correlationRules);
corrBuilder.endObject();

IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_INDEX)
IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_HISTORY_WRITE_INDEX)

Check warning on line 186 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L186 

Added line #L186 was not covered by tests
.source(corrBuilder)
.timeout(indexTimeout);
bulkRequest.add(indexRequest);
Expand Down Expand Up @@ -241,7 +241,7 @@ public void insertOrphanFindings(String detectorType, Finding finding, float tim
searchSourceBuilder.fetchSource(true);
searchSourceBuilder.size(1);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(CorrelationIndices.CORRELATION_INDEX);
searchRequest.indices(CorrelationIndices.CORRELATION_METADATA_INDEX);

Check warning on line 244 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L244 

Added line #L244 was not covered by tests
searchRequest.source(searchSourceBuilder);
searchRequest.preference(Preference.PRIMARY_FIRST.type());

Expand All @@ -268,7 +268,7 @@ public void onResponse(SearchResponse response) {
builder.field("scoreTimestamp", 0L);
builder.endObject();

IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_INDEX)
IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_METADATA_INDEX)

Check warning on line 271 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L271 

Added line #L271 was not covered by tests
.id(id)
.source(builder)
.timeout(indexTimeout)
Expand All @@ -295,7 +295,7 @@ public void onResponse(IndexResponse response) {
builder.field("scoreTimestamp", 0L);
builder.endObject();

IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_INDEX)
IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_HISTORY_WRITE_INDEX)

Check warning on line 298 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L298 

Added line #L298 was not covered by tests
.source(builder)
.timeout(indexTimeout)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
Expand Down Expand Up @@ -338,7 +338,7 @@ public void onFailure(Exception e) {
builder.field("scoreTimestamp", 0L);
builder.endObject();

IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_INDEX)
IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_METADATA_INDEX)

Check warning on line 341 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L341 

Added line #L341 was not covered by tests
.id(id)
.source(builder)
.timeout(indexTimeout)
Expand Down Expand Up @@ -366,7 +366,7 @@ public void onResponse(IndexResponse response) {
builder.field("scoreTimestamp", 0L);
builder.endObject();

IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_INDEX)
IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_HISTORY_WRITE_INDEX)

Check warning on line 369 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L369 

Added line #L369 was not covered by tests
.source(builder)
.timeout(indexTimeout)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
Expand Down Expand Up @@ -417,7 +417,7 @@ public void onFailure(Exception e) {
searchSourceBuilder.fetchSource(true);
searchSourceBuilder.size(1);
SearchRequest searchRequest = new SearchRequest();
searchRequest.indices(CorrelationIndices.CORRELATION_INDEX);
searchRequest.indices(CorrelationIndices.CORRELATION_HISTORY_INDEX_PATTERN_REGEXP);

Check warning on line 420 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L420 

Added line #L420 was not covered by tests
searchRequest.source(searchSourceBuilder);
searchRequest.preference(Preference.PRIMARY_FIRST.type());

Expand Down Expand Up @@ -458,7 +458,7 @@ public void onResponse(SearchResponse response) {
builder.field("scoreTimestamp", 0L);
builder.endObject();

IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_INDEX)
IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_HISTORY_WRITE_INDEX)

Check warning on line 461 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L461 

Added line #L461 was not covered by tests
.source(builder)
.timeout(indexTimeout)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
Expand Down Expand Up @@ -493,7 +493,7 @@ public void onFailure(Exception e) {
builder.field("scoreTimestamp", 0L);
builder.endObject();

IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_INDEX)
IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_METADATA_INDEX)

Check warning on line 496 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L496 

Added line #L496 was not covered by tests
.id(id)
.source(builder)
.timeout(indexTimeout)
Expand Down Expand Up @@ -523,7 +523,7 @@ public void onResponse(IndexResponse response) {
builder.field("scoreTimestamp", 0L);
builder.endObject();

IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_INDEX)
IndexRequest indexRequest = new IndexRequest(CorrelationIndices.CORRELATION_HISTORY_WRITE_INDEX)

Check warning on line 526 in src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java

View check run for this annotation

Codecov / codecov/patch

src/main/java/org/opensearch/securityanalytics/correlation/VectorEmbeddingsEngine.java#L526 

Added line #L526 was not covered by tests
.source(builder)
.timeout(indexTimeout)
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
Expand Down
Loading

0 comments on commit 259808f

Please sign in to comment.