Skip to content

Commit

Permalink
Backports for 2.5 (#927)
Browse files Browse the repository at this point in the history 
* query_field_names bugfix (#335)

Signed-off-by: Petar Dzepina <petar.dzepina@gmail.com>

* Reduce log level for informative message (#203)

Signed-off-by: Enrico Tröger <enrico.troeger@uvena.de>

* fix detector writeTo() method missing fields (#695)

* fix detector writeTo() method missing fields

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* fix test

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

---------

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* fix null query filter conversion from sigma to query string query (#722)

* fix null query filter conversion from sigma to query string query

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* fix rule to query conversion tests for null filter

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* enhance test to verify non null doc doesnt match null query

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

---------

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

* Bump version 2.5.1, fix build

Signed-off-by: Chase Engelbrecht <engechas@amazon.com>

* Manual cherry-pick of #873

Signed-off-by: Chase Engelbrecht <engechas@amazon.com>

* Fix miss from manual cherry-pick

Signed-off-by: Chase Engelbrecht <engechas@amazon.com>

* Undo exceptional case not originally present

Signed-off-by: Chase Engelbrecht <engechas@amazon.com>

* Revert test from previous commit reversion

Signed-off-by: Chase Engelbrecht <engechas@amazon.com>

* Fix another miss from manual cherry-pick

Signed-off-by: Chase Engelbrecht <engechas@amazon.com>

* Revert behavior for no rules in detector

Signed-off-by: Chase Engelbrecht <engechas@amazon.com>

---------

Signed-off-by: Petar Dzepina <petar.dzepina@gmail.com>
Signed-off-by: Enrico Tröger <enrico.troeger@uvena.de>
Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>
Signed-off-by: Chase Engelbrecht <engechas@amazon.com>
Co-authored-by: Petar Dzepina <petar.dzepina@gmail.com>
Co-authored-by: Enrico Tröger <enrico.troeger@uvena.de>
Co-authored-by: Surya Sashank Nistala <snistala@amazon.com>
  • Loading branch information
@engechas @petardz
@eht16 @eirsep
4 people authored Mar 15, 2024
1 parent 0025b7a commit 2d10915
Show file tree
Hide file tree
Showing 12 changed files with 535 additions and 207 deletions.
62 changes: 18 additions & 44 deletions build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask

buildscript {
ext {
opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT")
opensearch_version = System.getProperty("opensearch.version", "2.5.1-SNAPSHOT")
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
version_tokens = opensearch_version.tokenize('-')
Expand Down Expand Up @@ -125,6 +125,9 @@ repositories {
}

sourceSets.main.java.srcDirs = ['src/main/generated','src/main/java']
configurations {
zipArchive
}

dependencies {
javaRestTestImplementation project.sourceSets.main.runtimeClasspath
Expand All @@ -134,6 +137,11 @@ dependencies {
api "org.opensearch:common-utils:${common_utils_version}"
api "org.opensearch.client:opensearch-rest-client:${opensearch_version}"
implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"

// Needed for integ tests
zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}"
zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}"
zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}"
}

// RPM & Debian build
Expand Down Expand Up @@ -209,15 +217,6 @@ integTest.getClusters().forEach{c -> {
c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile()))
}}

String alertingFilePath = "src/test/resources/alerting"
String alertingPlugin = "opensearch-alerting-" + plugin_no_snapshot + ".zip"
String alertingRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + alertingPlugin
String notificationsFilePath = "src/test/resources/notifications"
String notificationsCoreFilePath = "src/test/resources/notifications-core"
String notificationsPlugin = "opensearch-notifications-" + plugin_no_snapshot + ".zip"
String notificationsCorePlugin = "opensearch-notifications-core-" + plugin_no_snapshot + ".zip"
String notificationsRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsPlugin
String notificationsCoreRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsCorePlugin
testClusters.integTest {
testDistribution = 'ARCHIVE'

Expand All @@ -233,58 +232,33 @@ testClusters.integTest {
debugPort += 1
}
}
setting 'path.repo', repo.absolutePath
plugin(provider({
new RegularFile() {
@Override
File getAsFile() {
File dir = new File(rootDir.path + "/" + alertingFilePath)

if (!dir.exists()) {
dir.mkdirs()
}

File f = new File(dir, alertingPlugin)
if (!f.exists()) {
new URL(alertingRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
}
fileTree(alertingFilePath).getSingleFile()
return configurations.zipArchive.asFileTree.matching {
include '**/opensearch-notifications-core*'
}.singleFile
}
}
}))
plugin(provider({
new RegularFile() {
@Override
File getAsFile() {
File dir = new File(rootDir.path + "/" + notificationsCoreFilePath)

if (!dir.exists()) {
dir.mkdirs()
}

File f = new File(dir, notificationsCorePlugin)
if (!f.exists()) {
new URL(notificationsCoreRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
}
fileTree(notificationsCoreFilePath).getSingleFile()
return configurations.zipArchive.asFileTree.matching {
include '**/notifications*'
}.singleFile
}
}
}))
plugin(provider({
new RegularFile() {
@Override
File getAsFile() {
File dir = new File(rootDir.path + "/" + notificationsFilePath)

if (!dir.exists()) {
dir.mkdirs()
}

File f = new File(dir, notificationsPlugin)
if (!f.exists()) {
new URL(notificationsRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }}
}
fileTree(notificationsFilePath).getSingleFile()
return configurations.zipArchive.asFileTree.matching {
include '**/alerting*'
}.singleFile
}
}
}))
Expand Down
2 changes: 1 addition & 1 deletion .../java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,7 @@ private String executorName() {
}

private void deleteOldIndices(String tag, String... indices) {
logger.error("info deleteOldIndices");
logger.info("info deleteOldIndices");
ClusterStateRequest clusterStateRequest = new ClusterStateRequest()
.clear()
.indices(indices)
Expand Down
6 changes: 5 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,11 @@ public void createMappingAction(String indexName, String ruleTopic, String alias
// since you can't update documents in non-write indices
String index = indexName;
boolean shouldUpsertIndexTemplate = IndexUtils.isConcreteIndex(indexName, this.clusterService.state()) == false;
if (IndexUtils.isDataStream(indexName, this.clusterService.state())) {
if (IndexUtils.isDataStream(indexName, this.clusterService.state()) || IndexUtils.isAlias(indexName, this.clusterService.state())) {
log.debug("{} is an alias or datastream. Fetching write index for create mapping action.", indexName);
String writeIndex = IndexUtils.getWriteIndex(indexName, this.clusterService.state());
if (writeIndex != null) {
log.debug("Write index for {} is {}", indexName, writeIndex);
index = writeIndex;
}
}
Expand All @@ -85,6 +87,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
applyAliasMappings(getMappingsResponse.getMappings(), ruleTopic, aliasMappings, partial, new ActionListener<>() {
@Override
public void onResponse(Collection<CreateMappingResult> createMappingResponse) {
log.debug("Completed create mappings for {}", indexName);
// We will return ack==false if one of the requests returned that
// else return ack==true
Optional<AcknowledgedResponse> notAckd = createMappingResponse.stream()
Expand All @@ -103,6 +106,7 @@ public void onResponse(Collection<CreateMappingResult> createMappingResponse) {

@Override
public void onFailure(Exception e) {
log.debug("Failed to create mappings for {}", indexName );
actionListener.onFailure(e);
}
});
Expand Down
20 changes: 12 additions & 8 deletions src/main/java/org/opensearch/securityanalytics/model/Detector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,12 +159,12 @@ public Detector(StreamInput sin) throws IOException {
sin.readList(DetectorInput::readFrom),
sin.readList(DetectorTrigger::readFrom),
sin.readStringList(),
sin.readString(),
sin.readString(),
sin.readString(),
sin.readString(),
sin.readString(),
sin.readString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readOptionalString(),
sin.readMap(StreamInput::readString, StreamInput::readString)
);
}
Expand Down Expand Up @@ -197,8 +197,12 @@ public void writeTo(StreamOutput out) throws IOException {
it.writeTo(out);
}
out.writeStringCollection(monitorIds);
out.writeString(ruleIndex);

out.writeOptionalString(ruleIndex);
out.writeOptionalString(alertsIndex);
out.writeOptionalString(alertsHistoryIndex);
out.writeOptionalString(alertsHistoryIndexPattern);
out.writeOptionalString(findingsIndex);
out.writeOptionalString(findingsIndexPattern);
out.writeMap(ruleIdMonitorIdMap, StreamOutput::writeString, StreamOutput::writeString);
}

Expand Down
4 changes: 3 additions & 1 deletion src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ public OSQueryBackend(String ruleCategory, boolean collectErrors, boolean enable
this.reEscapeChar = "\\";
this.reExpression = "%s: /%s/";
this.cidrExpression = "%s: \"%s\"";
this.fieldNullExpression = "%s: null";
this.fieldNullExpression = "%s: (NOT [* TO *])";
this.unboundValueStrExpression = "\"%s\"";
this.unboundValueNumExpression = "\"%s\"";
this.unboundWildcardExpression = "%s";
Expand Down Expand Up @@ -331,6 +331,8 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre

@Override
public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError {
String field = getFinalValueField();
ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer"));
SigmaString value = (SigmaString) condition.getValue();
boolean containsWildcard = value.containsWildcard();
return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue()));
Expand Down
4 changes: 4 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,10 @@ public Map<String, Object> getQueryFields() {
return queryFields;
}

public void resetQueryFields() {
queryFields.clear();
}

public abstract Object convertConditionAsInExpression(Either<ConditionAND, ConditionOR> condition);

public abstract Object convertConditionAnd(ConditionAND condition);
Expand Down
Loading

0 comments on commit 2d10915

Please sign in to comment.