Skip to content

Commit

Permalink
Adding various OCSF 1.1 fields to log type static mappings (#1403)
Browse files Browse the repository at this point in the history 
* Adding various OCSF 1.1 fields to log type static mappings

Signed-off-by: Dennis Toepker <toepkerd@amazon.com>

* fixing IT failures

Signed-off-by: Dennis Toepker <toepkerd@amazon.com>

* removed vestigial exception throw

Signed-off-by: Dennis Toepker <toepkerd@amazon.com>

* turning all ocsf 1.0 replacements with additions

Signed-off-by: Dennis Toepker <toepkerd@amazon.com>

* fixed ITs

Signed-off-by: Dennis Toepker <toepkerd@amazon.com>

---------

Signed-off-by: Dennis Toepker <toepkerd@amazon.com>
Co-authored-by: Dennis Toepker <toepkerd@amazon.com>
(cherry picked from commit 189b9e5)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
  • Loading branch information
@github-actions @toepkerd-zz
github-actions[bot] and toepkerd-zz committed Jan 8, 2025
1 parent 28c515f commit 292b9d8
Show file tree
Hide file tree
Showing 13 changed files with 202 additions and 93 deletions.
9 changes: 7 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/logtype/LogTypeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -566,6 +566,9 @@ private List<FieldMappingDoc> createFieldMappingDocs(List<LogType> logTypes) {
if (mapping.getOcsf() != null) {
schemaFields.put("ocsf", mapping.getOcsf());
}
if (mapping.getOcsf11() != null) {
schemaFields.put("ocsf11", mapping.getOcsf11());
}
fieldMappingMap.put(
key,
new FieldMappingDoc(
Expand All @@ -577,6 +580,7 @@ private List<FieldMappingDoc> createFieldMappingDocs(List<LogType> logTypes) {
} else {
// merge with existing doc
existingDoc.getSchemaFields().put("ocsf", mapping.getOcsf());
existingDoc.getSchemaFields().put("ocsf11", mapping.getOcsf11());
existingDoc.getLogTypes().add(logType.getName());
}
}));
Expand Down Expand Up @@ -705,7 +709,7 @@ public void getRuleFieldMappingsAllSchemas(String logType, ActionListener<List<L
(delegatedListener, fieldMappingDocs) -> {
List<LogType.Mapping> ruleFieldMappings = new ArrayList<>();
fieldMappingDocs.forEach( e -> {
ruleFieldMappings.add(new LogType.Mapping(e.getRawField(), e.getSchemaFields().get("ecs"), e.getSchemaFields().get("ocsf")));
ruleFieldMappings.add(new LogType.Mapping(e.getRawField(), e.getSchemaFields().get("ecs"), e.getSchemaFields().get("ocsf"), e.getSchemaFields().get("ocsf11")));
});
delegatedListener.onResponse(ruleFieldMappings);
}
Expand All @@ -728,7 +732,8 @@ public void getRequiredFields(String logType, ActionListener<List<LogType.Mappin
LogType.Mapping requiredField = new LogType.Mapping(
e.getRawField(),
e.getSchemaFields().get(defaultSchemaField),
e.getSchemaFields().get("ocsf")
e.getSchemaFields().get("ocsf"),
e.getSchemaFields().get("ocsf11")
);
requiredFields.add(requiredField);
});
Expand Down
8 changes: 8 additions & 0 deletions src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,8 @@ public void onResponse(List<LogType.Mapping> mappings) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getRawField()));
} else if (indexFields.contains(mapping.getOcsf())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
} else if (indexFields.contains(mapping.getOcsf11())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11()));
}
}
aliasMappingsObj.field("properties", aliasMappingFields);
Expand Down Expand Up @@ -483,6 +485,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
String alias = requiredField.getEcs();
String rawPath = requiredField.getRawField();
String ocsfPath = requiredField.getOcsf();
String ocsf11Path = requiredField.getOcsf11();
if (allFieldsFromIndex.contains(rawPath)) {
// if the alias was already added into applyable aliases, then skip to avoid duplicates
if (!applyableAliases.contains(alias) && !applyableAliases.contains(rawPath)) {
Expand All @@ -497,6 +500,9 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
} else if (allFieldsFromIndex.contains(ocsfPath)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsfPath);
} else if (allFieldsFromIndex.contains(ocsf11Path)) {
applyableAliases.add(alias);
pathsOfApplyableAliases.add(ocsf11Path);
} else if ((alias == null && allFieldsFromIndex.contains(rawPath) == false) || allFieldsFromIndex.contains(alias) == false) {
if (alias != null) {
// we don't want to send back aliases which have same name as existing field in index
Expand All @@ -520,6 +526,8 @@ public void onResponse(GetMappingsResponse getMappingsResponse) {
for (LogType.Mapping mapping : requiredFields) {
if (allFieldsFromIndex.contains(mapping.getOcsf())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf()));
} else if (allFieldsFromIndex.contains(mapping.getOcsf11())) {
aliasMappingFields.put(mapping.getEcs(), Map.of("type", "alias", "path", mapping.getOcsf11()));
} else if (mapping.getEcs() != null) {
shouldUpdateEcsMappingAndMaybeUpdates(mapping, aliasMappingFields, pathsOfApplyableAliases);
} else if (mapping.getEcs() == null) {
Expand Down
11 changes: 9 additions & 2 deletions src/main/java/org/opensearch/securityanalytics/model/LogType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ public class LogType implements Writeable {
private static final String RAW_FIELD = "raw_field";
public static final String ECS = "ecs";
public static final String OCSF = "ocsf";
public static final String OCSF11 = "ocsf11";
public static final String IOC_FIELDS = "ioc_fields";
public static final String IOC = "ioc";
public static final String FIELDS = "fields";
Expand Down Expand Up @@ -67,7 +68,7 @@ public LogType(Map<String, Object> logTypeAsMap) {
if (mappings.size() > 0) {
this.mappings = new ArrayList<>(mappings.size());
this.mappings = mappings.stream().map(e ->
new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF))
new Mapping(e.get(RAW_FIELD), e.get(ECS), e.get(OCSF), e.get(OCSF11))
).collect(Collectors.toList());
}
if (logTypeAsMap.containsKey(IOC_FIELDS)) {
Expand Down Expand Up @@ -120,17 +121,20 @@ public static class Mapping implements Writeable {
private String rawField;
private String ecs;
private String ocsf;
private String ocsf11;

public Mapping(StreamInput sin) throws IOException {
this.rawField = sin.readString();
this.ecs = sin.readOptionalString();
this.ocsf = sin.readOptionalString();
this.ocsf11 = sin.readOptionalString();
}

public Mapping(String rawField, String ecs, String ocsf) {
public Mapping(String rawField, String ecs, String ocsf, String ocsf11) {
this.rawField = rawField;
this.ecs = ecs;
this.ocsf = ocsf;
this.ocsf11 = ocsf11;
}

public String getRawField() {
Expand All @@ -145,11 +149,14 @@ public String getOcsf() {
return ocsf;
}

public String getOcsf11() { return ocsf11; }

@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(rawField);
out.writeOptionalString(ecs);
out.writeOptionalString(ocsf);
out.writeOptionalString(ocsf11);
}

public static Mapping readFrom(StreamInput sin) throws IOException {
Expand Down
24 changes: 16 additions & 8 deletions src/main/resources/OSMapping/cloudtrail_logtype.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,8 @@
{
"raw_field":"eventType",
"ecs":"aws.cloudtrail.event_type",
"ocsf": "unmapped.eventType"
"ocsf" : "unmapped.eventType",
"ocsf11": "metadata.event_code"
},
{
"raw_field":"eventCategory",
Expand Down Expand Up @@ -69,7 +70,8 @@
{
"raw_field":"additionalEventData.MFAUsed",
"ecs":"aws.cloudtrail.additional_event_data.mfaUsed",
"ocsf": "mfa"
"ocsf": "mfa",
"ocsf11": "is_mfa"
},
{
"raw_field":"responseElements",
Expand Down Expand Up @@ -124,12 +126,14 @@
{
"raw_field":"requestParameters.userName",
"ecs":"aws.cloudtrail.request_parameters.username",
"ocsf": "unmapped.requestParameters.userName"
"ocsf": "unmapped.requestParameters.userName",
"ocsf11": "user.name"
},
{
"raw_field":"requestParameters.roleArn",
"ecs":"aws.cloudtrail.request_parameters.roleArn",
"ocsf": "user.uuid"
"ocsf": "user.uuid",
"ocsf11": "user.uid"
},
{
"raw_field":"requestParameters.roleSessionName",
Expand All @@ -149,17 +153,20 @@
{
"raw_field":"userIdentity.principalId",
"ecs":"aws.cloudtrail.user_identity.principalId",
"ocsf": "actor.user.uid"
"ocsf": "actor.user.uid",
"ocsf11":"actor.user.uid_alt"
},
{
"raw_field":"userIdentity.arn",
"ecs":"aws.cloudtrail.user_identity.arn",
"ocsf": "actor.user.uuid"
"ocsf": "actor.user.uuid",
"ocsf11": "actor.user.uid"
},
{
"raw_field":"userIdentity.accountId",
"ecs":"aws.cloudtrail.user_identity.accountId",
"ocsf": "actor.user.account_uid"
"ocsf": "actor.user.account_uid",
"ocsf11": "actor.user.account.uid"
},
{
"raw_field":"userIdentity.accessKeyId",
Expand Down Expand Up @@ -199,7 +206,8 @@
{
"raw_field":"userIdentity.sessionContext.attributes.mfaAuthenticated",
"ecs":"aws.cloudtrail.user_identity.session_context.attributes.mfaAuthenticated",
"ocsf": "actor.session.mfa"
"ocsf": "actor.session.mfa",
"ocsf11": "actor.session.is_mfa"
},
{
"raw_field":"userIdentity.webIdFederationData.federatedProvider",
Expand Down
16 changes: 14 additions & 2 deletions src/main/resources/OSMapping/dns_logtype.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,8 @@
{
"raw_field":"account_id",
"ecs":"aws.route53.account_id",
"ocsf": "cloud.account_uid"
"ocsf": "cloud.account_uid",
"ocsf11": "cloud.account.uid"
},
{
"raw_field":"region",
Expand Down Expand Up @@ -114,12 +115,23 @@
{
"raw_field":"firewall_rule_action",
"ecs":"aws.route53.srcids.firewall_rule_action",
"ocsf": "disposition_id"
"ocsf": "disposition_id",
"ocsf11": "disposition"
},
{
"raw_field":"creationTime",
"ecs":"timestamp",
"ocsf": "unmapped.creationTime"
},
{
"raw_field":"rcode",
"ecs":"aws.route53.rcode",
"ocsf":"rcode"
},
{
"raw_field":"firewall_rule_group_id",
"ecs":"aws.route53.srcids.firewall_rule_group_id",
"ocsf":"firewall_rule.uid"
}
]
}
Loading

0 comments on commit 292b9d8

Please sign in to comment.