From b94266a46c2c659155acb156e9a51a56806ae63b Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Mon, 24 Jun 2024 13:03:52 +0300 Subject: [PATCH 1/6] FieldField class, renders improvements --- uncoder-core/app/translator/core/const.py | 6 ++ .../app/translator/core/context_vars.py | 6 ++ .../app/translator/core/models/field.py | 16 +++ .../translator/core/models/query_container.py | 2 +- uncoder-core/app/translator/core/parser.py | 3 +- uncoder-core/app/translator/core/render.py | 97 +++++++++++-------- uncoder-core/app/translator/core/tokenizer.py | 20 +++- .../platforms/athena/renders/athena.py | 6 +- .../platforms/base/aql/renders/aql.py | 6 +- .../platforms/base/lucene/renders/lucene.py | 4 +- .../platforms/base/spl/renders/spl.py | 4 +- .../platforms/base/sql/renders/sql.py | 4 +- .../platforms/chronicle/renders/chronicle.py | 6 +- .../chronicle/renders/chronicle_rule.py | 6 +- .../crowdstrike/renders/crowdstrike.py | 6 +- .../elasticsearch/renders/detection_rule.py | 7 +- .../elasticsearch/renders/elast_alert.py | 7 +- .../elasticsearch/renders/elasticsearch.py | 6 +- .../platforms/elasticsearch/renders/kibana.py | 7 +- .../elasticsearch/renders/xpack_watcher.py | 7 +- .../forti_siem/renders/forti_siem_rule.py | 16 ++- .../platforms/graylog/renders/graylog.py | 6 +- .../platforms/hunters/renders/hunters.py | 6 +- .../renders/logrhythm_axon_query.py | 24 ++--- .../renders/logrhythm_axon_rule.py | 11 +-- .../platforms/logscale/renders/logscale.py | 13 +-- .../logscale/renders/logscale_alert.py | 13 +-- .../microsoft/renders/microsoft_defender.py | 6 +- .../microsoft/renders/microsoft_sentinel.py | 6 +- .../renders/microsoft_sentinel_rule.py | 11 +-- .../opensearch/renders/opensearch.py | 6 +- .../opensearch/renders/opensearch_rule.py | 11 +-- .../palo_alto/renders/cortex_xsiam.py | 23 ++++- .../platforms/qradar/renders/qradar.py | 5 +- .../platforms/sigma/renders/sigma.py | 8 +- .../platforms/splunk/renders/splunk.py | 6 +- .../platforms/splunk/renders/splunk_alert.py | 11 +-- 37 files changed, 221 insertions(+), 187 deletions(-) create mode 100644 uncoder-core/app/translator/core/const.py diff --git a/uncoder-core/app/translator/core/const.py b/uncoder-core/app/translator/core/const.py new file mode 100644 index 00000000..a8788ada --- /dev/null +++ b/uncoder-core/app/translator/core/const.py @@ -0,0 +1,6 @@ +from typing import Union + +from app.translator.core.models.field import Alias, Field, FieldValue, Keyword +from app.translator.core.models.identifier import Identifier + +TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field, Alias] diff --git a/uncoder-core/app/translator/core/context_vars.py b/uncoder-core/app/translator/core/context_vars.py index 591883d8..8ff6ccfb 100644 --- a/uncoder-core/app/translator/core/context_vars.py +++ b/uncoder-core/app/translator/core/context_vars.py @@ -1,4 +1,10 @@ from contextvars import ContextVar +from typing import Optional return_only_first_query_ctx_var: ContextVar[bool] = ContextVar("return_only_first_query_ctx_var", default=False) """Set to True to return only first query if rendered multiple options""" + +wrap_query_with_meta_info_ctx_var: ContextVar[bool] = ContextVar("wrap_query_with_meta_info_ctx_var", default=True) +"""Set to False not to wrap query with meta info commentary""" + +preset_log_source_str_ctx_var: ContextVar[Optional[str]] = ContextVar("preset_log_source_str_ctx_var", default=None) diff --git a/uncoder-core/app/translator/core/models/field.py b/uncoder-core/app/translator/core/models/field.py index 10b661b0..0c430382 100644 --- a/uncoder-core/app/translator/core/models/field.py +++ b/uncoder-core/app/translator/core/models/field.py @@ -37,6 +37,22 @@ def set_generic_names_map(self, source_mappings: list[SourceMapping], default_ma self.__generic_names_map = generic_names_map +class FieldField: + def __init__( + self, + source_name_left: str, + operator: Identifier, + source_name_right: str, + is_alias_left: bool = False, + is_alias_right: bool = False, + ): + self.field_left = Field(source_name=source_name_left) + self.alias_left = Alias(name=source_name_left) if is_alias_left else None + self.operator = operator + self.field_right = Field(source_name=source_name_right) + self.alias_right = Alias(name=source_name_right) if is_alias_right else None + + class FieldValue: def __init__( self, diff --git a/uncoder-core/app/translator/core/models/query_container.py b/uncoder-core/app/translator/core/models/query_container.py index dccfc180..0d90f237 100644 --- a/uncoder-core/app/translator/core/models/query_container.py +++ b/uncoder-core/app/translator/core/models/query_container.py @@ -3,11 +3,11 @@ from datetime import datetime from typing import Optional +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.mapping import DEFAULT_MAPPING_NAME from app.translator.core.models.field import Field from app.translator.core.models.functions.base import ParsedFunctions -from app.translator.core.tokenizer import TOKEN_TYPE class MetaInfoContainer: diff --git a/uncoder-core/app/translator/core/parser.py b/uncoder-core/app/translator/core/parser.py index 7cc10ec1..18b50739 100644 --- a/uncoder-core/app/translator/core/parser.py +++ b/uncoder-core/app/translator/core/parser.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.exceptions.parser import TokenizerGeneralException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import BasePlatformMappings, SourceMapping @@ -28,7 +29,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import RawQueryContainer, TokenizedQueryContainer -from app.translator.core.tokenizer import TOKEN_TYPE, QueryTokenizer +from app.translator.core.tokenizer import QueryTokenizer class QueryParser(ABC): diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index bf28b4f6..636f718f 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -16,13 +16,14 @@ limitations under the License. ----------------------------------------------------------------- """ - +import itertools from abc import ABC, abstractmethod from collections.abc import Callable from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE -from app.translator.core.context_vars import return_only_first_query_ctx_var +from app.translator.core.const import TOKEN_TYPE +from app.translator.core.context_vars import return_only_first_query_ctx_var, wrap_query_with_meta_info_ctx_var from app.translator.core.custom_types.tokens import LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -30,22 +31,21 @@ from app.translator.core.exceptions.parser import UnsupportedOperatorException from app.translator.core.functions import PlatformFunctions from app.translator.core.mapping import DEFAULT_MAPPING_NAME, BasePlatformMappings, LogSourceSignature, SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function, RenderedFunctions from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, RawQueryContainer, TokenizedQueryContainer from app.translator.core.str_value_manager import StrValue, StrValueManager -from app.translator.core.tokenizer import TOKEN_TYPE -class BaseQueryFieldValue(ABC): +class BaseFieldValueRender(ABC): details: PlatformDetails = None escape_manager: EscapeManager = None str_value_manager: StrValueManager = None def __init__(self, or_token: str): - self.field_value: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { + self.modifiers_map: dict[str, Callable[[str, DEFAULT_VALUE_TYPE], str]] = { OperatorType.EQ: self.equal_modifier, OperatorType.NOT_EQ: self.not_equal_modifier, OperatorType.LT: self.less_modifier, @@ -155,11 +155,20 @@ def apply_value(self, value: Union[str, int], value_type: str = ValueType.value) return self.escape_manager.escape(value, value_type) def apply_field_value(self, field: str, operator: Identifier, value: DEFAULT_VALUE_TYPE) -> str: - if modifier_function := self.field_value.get(operator.token_type): + if modifier_function := self.modifiers_map.get(operator.token_type): return modifier_function(field, value) raise UnsupportedOperatorException(operator.token_type) +class BaseFieldFieldRender(ABC): + operators_map: ClassVar[dict[str, str]] = {} + + def apply_field_field(self, field_left: str, operator: Identifier, field_right: str) -> str: + if mapped_operator := self.operators_map.get(operator.token_type): + return f"{field_left} {mapped_operator} {field_right}" + raise UnsupportedOperatorException(operator.token_type) + + class QueryRender(ABC): comment_symbol: str = None details: PlatformDetails = None @@ -180,6 +189,13 @@ def render_not_supported_functions(self, not_supported_functions: list) -> str: not_supported_functions_str = "\n".join(line_template + func.lstrip() for func in not_supported_functions) return "\n\n" + self.wrap_with_comment(f"{self.unsupported_functions_text}\n{not_supported_functions_str}") + def wrap_with_not_supported_functions(self, query: str, not_supported_functions: Optional[list] = None) -> str: + if not_supported_functions and wrap_query_with_meta_info_ctx_var.get(): + rendered_not_supported = self.render_not_supported_functions(not_supported_functions) + return query + rendered_not_supported + + return query + def wrap_with_comment(self, value: str) -> str: return f"{self.comment_symbol} {value}" @@ -199,13 +215,14 @@ class PlatformQueryRender(QueryRender): group_token = "(%s)" query_parts_delimiter = " " - field_value_map = BaseQueryFieldValue(or_token=or_token) + field_field_render = BaseFieldFieldRender() + field_value_render = BaseFieldValueRender(or_token=or_token) raw_log_field_pattern_map: ClassVar[dict[str, str]] = None def __init__(self): super().__init__() - self.operator_map = { + self.logical_operators_map = { LogicalOperatorType.AND: f" {self.and_token} ", LogicalOperatorType.OR: f" {self.or_token} ", LogicalOperatorType.NOT: f" {self.not_token} ", @@ -233,31 +250,34 @@ def map_field(self, field: Field, source_mapping: SourceMapping) -> list[str]: def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): - if token.alias: - field_name = token.alias.name - else: - mapped_fields = self.map_field(token.field, source_mapping) - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value( - field=field, operator=token.operator, value=token.value - ) - for field in mapped_fields - ] - ) - - field_name = mapped_fields[0] - - return self.field_value_map.apply_field_value(field=field_name, operator=token.operator, value=token.value) - + mapped_fields = [token.alias.name] if token.alias else self.map_field(token.field, source_mapping) + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] + ) + return self.group_token % joined if len(mapped_fields) > 1 else joined + if isinstance(token, FieldField): + alias_left, field_left = token.alias_left, token.field_left + mapped_fields_left = [alias_left.name] if alias_left else self.map_field(field_left, source_mapping) + alias_right, field_right = token.alias_right, token.field_right + mapped_fields_right = [alias_right.name] if alias_right else self.map_field(field_right, source_mapping) + cross_paired_fields = list(itertools.product(mapped_fields_left, mapped_fields_right)) + joined = self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_field_render.apply_field_field(pair[0], token.operator, pair[1]) + for pair in cross_paired_fields + ] + ) + return self.group_token % joined if len(cross_paired_fields) > 1 else joined if isinstance(token, Function): func_render = self.platform_functions.manager.get_in_query_render(token.name) return func_render.render(token, source_mapping) if isinstance(token, Keyword): - return self.field_value_map.apply_field_value(field="", operator=token.operator, value=token.value) + return self.field_value_render.apply_field_value(field="", operator=token.operator, value=token.value) if token.token_type in LogicalOperatorType: - return self.operator_map.get(token.token_type) + return self.logical_operators_map.get(token.token_type) return token.token_type @@ -267,8 +287,8 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) return "".join(result_values) - def wrap_query_with_meta_info(self, meta_info: MetaInfoContainer, query: str) -> str: - if meta_info and (meta_info.id or meta_info.title): + def wrap_with_meta_info(self, query: str, meta_info: MetaInfoContainer) -> str: + if wrap_query_with_meta_info_ctx_var.get() and meta_info and (meta_info.id or meta_info.title): meta_info_dict = { "name: ": meta_info.title, "uuid: ": meta_info.id, @@ -301,11 +321,8 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = self._join_query_parts(prefix, query, functions) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) @staticmethod def unique_queries(queries_map: dict[str, str]) -> dict[str, dict[str]]: @@ -336,7 +353,7 @@ def _get_source_mappings(self, source_mapping_ids: list[str]) -> list[SourceMapp return source_mappings - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: return self.finalize_query( prefix="", query=query_container.query, functions="", meta_info=query_container.meta_info ) @@ -374,7 +391,7 @@ def generate_raw_log_fields(self, fields: list[Field], source_mapping: SourceMap defined_raw_log_fields.append(prefix) return "\n".join(defined_raw_log_fields) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} errors = [] source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -411,6 +428,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index 45486ef1..2ecbc39c 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -20,6 +20,7 @@ from abc import ABC, abstractmethod from typing import Any, ClassVar, Optional, Union +from app.translator.core.const import TOKEN_TYPE from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.escape_manager import EscapeManager @@ -29,18 +30,18 @@ UnsupportedOperatorException, ) from app.translator.core.mapping import SourceMapping -from app.translator.core.models.field import Field, FieldValue, Keyword +from app.translator.core.models.field import Field, FieldField, FieldValue, Keyword from app.translator.core.models.functions.base import Function from app.translator.core.models.functions.eval import EvalArg from app.translator.core.models.functions.group_by import GroupByFunction +from app.translator.core.models.functions.join import JoinFunction from app.translator.core.models.functions.rename import RenameArg from app.translator.core.models.functions.sort import SortArg +from app.translator.core.models.functions.union import UnionFunction from app.translator.core.models.identifier import Identifier from app.translator.core.str_value_manager import StrValue, StrValueManager from app.translator.tools.utils import get_match_group -TOKEN_TYPE = Union[FieldValue, Keyword, Identifier, Field] - class BaseTokenizer(ABC): @abstractmethod @@ -323,13 +324,18 @@ def filter_tokens( ) -> list[TOKEN_TYPE]: return [token for token in tokens if isinstance(token, token_type)] - def get_field_tokens_from_func_args( + def get_field_tokens_from_func_args( # noqa: PLR0912 self, args: list[Union[Field, FieldValue, Keyword, Identifier, Function, SortArg]] ) -> list[Field]: result = [] for arg in args: if isinstance(arg, Field): result.append(arg) + elif isinstance(arg, FieldField): + if not arg.alias_left or arg.alias_left.name != arg.field_left.source_name: + result.append(arg.field_left) + if not arg.alias_right or arg.alias_right.name != arg.field_right.source_name: + result.append(arg.field_right) elif isinstance(arg, FieldValue): if not arg.alias or arg.alias.name != arg.field.source_name: result.append(arg.field) @@ -337,6 +343,12 @@ def get_field_tokens_from_func_args( result.extend(self.get_field_tokens_from_func_args(args=arg.args)) result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_])) + elif isinstance(arg, (JoinFunction, UnionFunction)): + result.extend(self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.tokens)) + result.extend( + self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.functions.functions) + ) + result.extend(self.get_field_tokens_from_func_args(args=arg.condition)) elif isinstance(arg, Function): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) elif isinstance(arg, SortArg) and isinstance(arg.field, Field): diff --git a/uncoder-core/app/translator/platforms/athena/renders/athena.py b/uncoder-core/app/translator/platforms/athena/renders/athena.py index a62e5b00..8550c94a 100644 --- a/uncoder-core/app/translator/platforms/athena/renders/athena.py +++ b/uncoder-core/app/translator/platforms/athena/renders/athena.py @@ -21,10 +21,10 @@ from app.translator.managers import render_manager from app.translator.platforms.athena.const import athena_details from app.translator.platforms.athena.mapping import AthenaMappings, athena_mappings -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender -class AthenaFieldValue(SqlFieldValue): +class AthenaFieldValueRender(SqlFieldValueRender): details: PlatformDetails = athena_details @@ -35,7 +35,7 @@ class AthenaQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = AthenaFieldValue(or_token=or_token) + field_value_render = AthenaFieldValueRender(or_token=or_token) comment_symbol = "--" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py index 05826d08..6c0c1665 100644 --- a/uncoder-core/app/translator/platforms/base/aql/renders/aql.py +++ b/uncoder-core/app/translator/platforms/base/aql/renders/aql.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.aql.mapping import AQLLogSourceSignature, AQLMappings, aql_mappings from app.translator.platforms.base.aql.str_value_manager import aql_str_value_manager -class AQLFieldValue(BaseQueryFieldValue): +class AQLFieldValueRender(BaseFieldValueRender): str_value_manager = aql_str_value_manager @staticmethod @@ -127,8 +127,6 @@ class AQLQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = AQLFieldValue(or_token=or_token) - def generate_prefix(self, log_source_signature: AQLLogSourceSignature, functions_prefix: str = "") -> str: # noqa: ARG002 table = str(log_source_signature) extra_condition = log_source_signature.extra_condition diff --git a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py index b5994499..f8511d82 100644 --- a/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py +++ b/uncoder-core/app/translator/platforms/base/lucene/renders/lucene.py @@ -21,13 +21,13 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.custom_types.values import ValueType -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.platforms.base.lucene.mapping import LuceneLogSourceSignature from app.translator.platforms.base.lucene.str_value_manager import lucene_str_value_manager -class LuceneFieldValue(BaseQueryFieldValue): +class LuceneFieldValueRender(BaseFieldValueRender): str_value_manager = lucene_str_value_manager @staticmethod diff --git a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py index b2c12068..74adf32b 100644 --- a/uncoder-core/app/translator/platforms/base/spl/renders/spl.py +++ b/uncoder-core/app/translator/platforms/base/spl/renders/spl.py @@ -21,11 +21,11 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.platforms.base.spl.escape_manager import spl_escape_manager -class SplFieldValue(BaseQueryFieldValue): +class SplFieldValueRender(BaseFieldValueRender): escape_manager = spl_escape_manager def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: diff --git a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py index 43904a1e..d69f1590 100644 --- a/uncoder-core/app/translator/platforms/base/sql/renders/sql.py +++ b/uncoder-core/app/translator/platforms/base/sql/renders/sql.py @@ -22,10 +22,10 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.mapping import LogSourceSignature -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender -class SqlFieldValue(BaseQueryFieldValue): +class SqlFieldValueRender(BaseFieldValueRender): def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: if isinstance(value, list): return f"({self.or_token.join([self.equal_modifier(field=field, value=v) for v in value])})" diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py index 4101b825..8bcbe56f 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle.py @@ -23,14 +23,14 @@ from app.translator.core.custom_types.values import ValueType from app.translator.core.exceptions.render import UnsupportedRenderMethod from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import chronicle_query_details from app.translator.platforms.chronicle.escape_manager import chronicle_escape_manager from app.translator.platforms.chronicle.mapping import ChronicleMappings, chronicle_mappings -class ChronicleFieldValue(BaseQueryFieldValue): +class ChronicleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = chronicle_query_details escape_manager = chronicle_escape_manager @@ -109,6 +109,6 @@ class ChronicleQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = ChronicleFieldValue(or_token=or_token) + field_value_render = ChronicleFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = True diff --git a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py index aaa64384..1961c72b 100644 --- a/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py +++ b/uncoder-core/app/translator/platforms/chronicle/renders/chronicle_rule.py @@ -26,12 +26,12 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.chronicle.const import DEFAULT_CHRONICLE_SECURITY_RULE, chronicle_rule_details -from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValue, ChronicleQueryRender +from app.translator.platforms.chronicle.renders.chronicle import ChronicleFieldValueRender, ChronicleQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated Chronicle Security rule." -class ChronicleRuleFieldValue(ChronicleFieldValue): +class ChronicleRuleFieldValueRender(ChronicleFieldValueRender): details: PlatformDetails = chronicle_rule_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -85,7 +85,7 @@ def regex_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: class ChronicleSecurityRuleRender(ChronicleQueryRender): details: PlatformDetails = chronicle_rule_details or_token = "or" - field_value_map = ChronicleRuleFieldValue(or_token=or_token) + field_value_render = ChronicleRuleFieldValueRender(or_token=or_token) @staticmethod def prepare_title(title: str) -> str: diff --git a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py index 8c6630e9..3e5900cc 100644 --- a/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py +++ b/uncoder-core/app/translator/platforms/crowdstrike/renders/crowdstrike.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.crowdstrike.const import crowdstrike_query_details from app.translator.platforms.crowdstrike.functions import CrowdStrikeFunctions, crowd_strike_functions from app.translator.platforms.crowdstrike.mapping import CrowdstrikeMappings, crowdstrike_mappings -class CrowdStrikeFieldValue(SplFieldValue): +class CrowdStrikeFieldValueRender(SplFieldValueRender): details = crowdstrike_query_details @@ -36,7 +36,7 @@ class CrowdStrikeQueryRender(SplQueryRender): platform_functions: CrowdStrikeFunctions = None or_token = "OR" - field_value_map = CrowdStrikeFieldValue(or_token=or_token) + field_value_render = CrowdStrikeFieldValueRender(or_token=or_token) comment_symbol = "`" def init_platform_functions(self) -> None: diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py index 09fad79b..0b7b20c4 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/detection_rule.py @@ -50,7 +50,7 @@ class ElasticSearchRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticSearchRuleFieldValue(or_token=or_token) + field_value_render = ElasticSearchRuleFieldValue(or_token=or_token) def __create_mitre_threat(self, mitre_attack: dict) -> Union[list, list[dict]]: if not mitre_attack.get("techniques"): @@ -109,7 +109,4 @@ def finalize_query( } ) rule_str = json.dumps(rule, indent=4, sort_keys=False, ensure_ascii=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py index 104b8ecc..9d7914ab 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elast_alert.py @@ -49,7 +49,7 @@ class ElastAlertRuleRender(ElasticSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = ElasticAlertRuleFieldValue(or_token=or_token) + field_value_render = ElasticAlertRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -75,7 +75,4 @@ def finalize_query( ) rule = rule.replace("", meta_info.title or _AUTOGENERATED_TEMPLATE) rule = rule.replace("", _SEVERITIES_MAP[meta_info.severity]) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py index 8d2db1d0..2e6a12f0 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/elasticsearch.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.elasticsearch.const import elasticsearch_lucene_query_details from app.translator.platforms.elasticsearch.mapping import ElasticSearchMappings, elasticsearch_mappings -class ElasticSearchFieldValue(LuceneFieldValue): +class ElasticSearchFieldValue(LuceneFieldValueRender): details: PlatformDetails = elasticsearch_lucene_query_details @@ -34,4 +34,4 @@ class ElasticSearchQueryRender(LuceneQueryRender): mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = ElasticSearchFieldValue(or_token=or_token) + field_value_render = ElasticSearchFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py index c3b6a46a..53a4acf5 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/kibana.py @@ -45,7 +45,7 @@ class KibanaRuleRender(ElasticSearchQueryRender): details: PlatformDetails = kibana_rule_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = KibanaFieldValue(or_token=or_token) + field_value_render = KibanaFieldValue(or_token=or_token) def finalize_query( self, @@ -74,7 +74,4 @@ def finalize_query( references=meta_info.references, ) rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py index 9a013dcf..d8421977 100644 --- a/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py +++ b/uncoder-core/app/translator/platforms/elasticsearch/renders/xpack_watcher.py @@ -45,7 +45,7 @@ class XPackWatcherRuleRender(ElasticSearchQueryRender): details: PlatformDetails = xpack_watcher_details mappings: ElasticSearchMappings = elasticsearch_mappings or_token = "OR" - field_value_map = XpackWatcherRuleFieldValue(or_token=or_token) + field_value_render = XpackWatcherRuleFieldValue(or_token=or_token) def finalize_query( self, @@ -78,7 +78,4 @@ def finalize_query( rule["input"]["search"]["request"]["indices"] = indices rule["actions"]["send_email"]["email"]["subject"] = meta_info.title or _AUTOGENERATED_TEMPLATE rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py index 65ca0b07..dfbc2ee6 100644 --- a/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py +++ b/uncoder-core/app/translator/platforms/forti_siem/renders/forti_siem_rule.py @@ -18,6 +18,7 @@ from typing import Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.const import TOKEN_TYPE from app.translator.core.context_vars import return_only_first_query_ctx_var from app.translator.core.custom_types.meta_info import SeverityType from app.translator.core.custom_types.tokens import GroupType, LogicalOperatorType, OperatorType @@ -28,9 +29,8 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer, TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue -from app.translator.core.tokenizer import TOKEN_TYPE from app.translator.managers import render_manager from app.translator.platforms.forti_siem.const import ( FORTI_SIEM_RULE, @@ -76,7 +76,7 @@ ] -class FortiSiemFieldValue(BaseQueryFieldValue): +class FortiSiemFieldValueRender(BaseFieldValueRender): details: PlatformDetails = forti_siem_rule_details str_value_manager = forti_siem_str_value_manager @@ -194,7 +194,7 @@ class FortiSiemRuleRender(PlatformQueryRender): group_token = "(%s)" - field_value_map = FortiSiemFieldValue(or_token=or_token) + field_value_render = FortiSiemFieldValueRender(or_token=or_token) @staticmethod def __is_negated_token(prev_token: TOKEN_TYPE) -> bool: @@ -244,7 +244,7 @@ def __replace_not_tokens(self, tokens: list[TOKEN_TYPE]) -> list[TOKEN_TYPE]: return tokens - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) @@ -324,11 +324,7 @@ def finalize_query( rule = rule.replace("", query) rule = rule.replace("", ", ".join(args_list)) rule = rule.replace("", self.get_attr_str(fields.copy())) - - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) @staticmethod def get_attr_str(fields: set[str]) -> str: diff --git a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py index 2bdf001e..986ddd93 100644 --- a/uncoder-core/app/translator/platforms/graylog/renders/graylog.py +++ b/uncoder-core/app/translator/platforms/graylog/renders/graylog.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.graylog.const import graylog_details from app.translator.platforms.graylog.mapping import GraylogMappings, graylog_mappings -class GraylogFieldValue(LuceneFieldValue): +class GraylogFieldValue(LuceneFieldValueRender): details: PlatformDetails = graylog_details @@ -34,4 +34,4 @@ class GraylogQueryRender(LuceneQueryRender): mappings: GraylogMappings = graylog_mappings or_token = "OR" - field_value_map = GraylogFieldValue(or_token=or_token) + field_value_render = GraylogFieldValue(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py index 0348bfb0..3c73c234 100644 --- a/uncoder-core/app/translator/platforms/hunters/renders/hunters.py +++ b/uncoder-core/app/translator/platforms/hunters/renders/hunters.py @@ -19,12 +19,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.sql.renders.sql import SqlFieldValue, SqlQueryRender +from app.translator.platforms.base.sql.renders.sql import SqlFieldValueRender, SqlQueryRender from app.translator.platforms.hunters.const import hunters_details from app.translator.platforms.hunters.mapping import HuntersMappings, hunters_mappings -class HuntersFieldValue(SqlFieldValue): +class HuntersFieldValueRender(SqlFieldValueRender): details: PlatformDetails = hunters_details @@ -35,7 +35,7 @@ class HuntersQueryRender(SqlQueryRender): or_token = "OR" - field_value_map = HuntersFieldValue(or_token=or_token) + field_value_render = HuntersFieldValueRender(or_token=or_token) @staticmethod def _finalize_search_query(query: str) -> str: diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 624fa3d7..0cef42f2 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -30,7 +30,7 @@ from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import TokenizedQueryContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logrhythm_axon.const import UNMAPPED_FIELD_DEFAULT_NAME, logrhythm_axon_query_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_query_escape_manager @@ -41,7 +41,7 @@ class LogRhythmRegexRenderException(BaseRenderException): ... -class LogRhythmAxonFieldValue(BaseQueryFieldValue): +class LogRhythmAxonFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logrhythm_axon_query_details escape_manager = logrhythm_query_escape_manager @@ -204,7 +204,7 @@ class LogRhythmAxonQueryRender(PlatformQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = LogRhythmAxonFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonFieldValueRender(or_token=or_token) mappings: LogRhythmAxonMappings = logrhythm_axon_mappings comment_symbol = "//" @@ -224,7 +224,7 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp mapped_fields = self.map_field(token.field, source_mapping) except StrictPlatformException: try: - return self.field_value_map.apply_field_value( + return self.field_value_render.apply_field_value( field=UNMAPPED_FIELD_DEFAULT_NAME, operator=token.operator, value=token.value ) except LogRhythmRegexRenderException as exc: @@ -232,20 +232,16 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp f"Uncoder does not support complex regexp for unmapped field:" f" {token.field.source_name} for LogRhythm Axon" ) from exc - if len(mapped_fields) > 1: - return self.group_token % self.operator_map[LogicalOperatorType.OR].join( - [ - self.field_value_map.apply_field_value(field=field, operator=token.operator, value=token.value) - for field in mapped_fields - ] - ) - return self.field_value_map.apply_field_value( - field=mapped_fields[0], operator=token.operator, value=token.value + return self.group_token % self.logical_operators_map[LogicalOperatorType.OR].join( + [ + self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) + for field in mapped_fields + ] ) return super().apply_token(token, source_mapping) - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: queries_map = {} source_mappings = self._get_source_mappings(query_container.meta_info.source_mapping_ids) diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py index 20514140..2e68c2d1 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_rule.py @@ -29,7 +29,7 @@ from app.translator.platforms.logrhythm_axon.const import DEFAULT_LOGRHYTHM_AXON_RULE, logrhythm_axon_rule_details from app.translator.platforms.logrhythm_axon.escape_manager import logrhythm_rule_escape_manager from app.translator.platforms.logrhythm_axon.renders.logrhythm_axon_query import ( - LogRhythmAxonFieldValue, + LogRhythmAxonFieldValueRender, LogRhythmAxonQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -44,7 +44,7 @@ } -class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): +class LogRhythmAxonRuleFieldValueRender(LogRhythmAxonFieldValueRender): details: PlatformDetails = logrhythm_axon_rule_details escape_manager = logrhythm_rule_escape_manager @@ -53,7 +53,7 @@ class LogRhythmAxonRuleFieldValue(LogRhythmAxonFieldValue): class LogRhythmAxonRuleRender(LogRhythmAxonQueryRender): details: PlatformDetails = logrhythm_axon_rule_details or_token = "or" - field_value_map = LogRhythmAxonRuleFieldValue(or_token=or_token) + field_value_render = LogRhythmAxonRuleFieldValueRender(or_token=or_token) def finalize_query( self, @@ -93,7 +93,4 @@ def finalize_query( ] json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py index 9cb7cf05..e1ed4818 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale.py @@ -23,7 +23,7 @@ from app.translator.core.mapping import SourceMapping from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.models.query_container import MetaInfoContainer -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.logscale.const import logscale_query_details from app.translator.platforms.logscale.escape_manager import logscale_escape_manager @@ -31,7 +31,7 @@ from app.translator.platforms.logscale.mapping import LogScaleMappings, logscale_mappings -class LogScaleFieldValue(BaseQueryFieldValue): +class LogScaleFieldValueRender(BaseFieldValueRender): details: PlatformDetails = logscale_query_details escape_manager = logscale_escape_manager @@ -102,7 +102,7 @@ class LogScaleQueryRender(PlatformQueryRender): and_token = "" not_token = "not" - field_value_map = LogScaleFieldValue(or_token=or_token) + field_value_render = LogScaleFieldValueRender(or_token=or_token) def init_platform_functions(self) -> None: self.platform_functions = log_scale_functions @@ -123,8 +123,5 @@ def finalize_query( **kwargs, # noqa: ARG002 ) -> str: query = super().finalize_query(prefix=prefix, query=query, functions=functions) - query = self.wrap_query_with_meta_info(meta_info=meta_info, query=query) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return query + rendered_not_supported - return query + query = self.wrap_with_meta_info(query, meta_info) + return self.wrap_with_not_supported_functions(query, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py index 4b3af0fb..a6628045 100644 --- a/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py +++ b/uncoder-core/app/translator/platforms/logscale/renders/logscale_alert.py @@ -26,13 +26,13 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.logscale.const import DEFAULT_LOGSCALE_ALERT, logscale_alert_details -from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValue, LogScaleQueryRender +from app.translator.platforms.logscale.renders.logscale import LogScaleFieldValueRender, LogScaleQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Falcon LogScale Alert" -class LogScaleAlertFieldValue(LogScaleFieldValue): +class LogScaleAlertFieldValueRender(LogScaleFieldValueRender): details: PlatformDetails = logscale_alert_details @@ -40,7 +40,7 @@ class LogScaleAlertFieldValue(LogScaleFieldValue): class LogScaleAlertRender(LogScaleQueryRender): details: PlatformDetails = logscale_alert_details or_token = "or" - field_value_map = LogScaleAlertFieldValue(or_token=or_token) + field_value_render = LogScaleAlertFieldValueRender(or_token=or_token) def finalize_query( self, @@ -70,8 +70,5 @@ def finalize_query( mitre_attack=mitre_attack, ) - json_query = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_query + rendered_not_supported - return json_query + rule_str = json.dumps(rule, indent=4, sort_keys=False) + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py index 7b7a3779..38617b55 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_defender.py @@ -23,12 +23,12 @@ from app.translator.platforms.microsoft.functions import MicrosoftFunctions, microsoft_defender_functions from app.translator.platforms.microsoft.mapping import MicrosoftDefenderMappings, microsoft_defender_mappings from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) -class MicrosoftDefenderFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftDefenderFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_defender_details @@ -38,7 +38,7 @@ class MicrosoftDefenderQueryRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_defender_details platform_functions: MicrosoftFunctions = None or_token = "or" - field_value_map = MicrosoftDefenderFieldValue(or_token=or_token) + field_value_render = MicrosoftDefenderFieldValueRender(or_token=or_token) is_strict_mapping = True diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py index 3153f8d4..7ef6f1f9 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel.py @@ -22,7 +22,7 @@ from app.translator.const import DEFAULT_VALUE_TYPE from app.translator.core.mapping import LogSourceSignature from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldValueRender, PlatformQueryRender from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import microsoft_sentinel_query_details from app.translator.platforms.microsoft.escape_manager import microsoft_escape_manager @@ -30,7 +30,7 @@ from app.translator.platforms.microsoft.mapping import MicrosoftSentinelMappings, microsoft_sentinel_mappings -class MicrosoftSentinelFieldValue(BaseQueryFieldValue): +class MicrosoftSentinelFieldValueRender(BaseFieldValueRender): details: PlatformDetails = microsoft_sentinel_query_details escape_manager = microsoft_escape_manager @@ -128,7 +128,7 @@ class MicrosoftSentinelQueryRender(PlatformQueryRender): and_token = "and" not_token = "not" - field_value_map = MicrosoftSentinelFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelFieldValueRender(or_token=or_token) mappings: MicrosoftSentinelMappings = microsoft_sentinel_mappings comment_symbol = "//" diff --git a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py index e2fdb81f..b5631ef5 100644 --- a/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py +++ b/uncoder-core/app/translator/platforms/microsoft/renders/microsoft_sentinel_rule.py @@ -28,7 +28,7 @@ from app.translator.managers import render_manager from app.translator.platforms.microsoft.const import DEFAULT_MICROSOFT_SENTINEL_RULE, microsoft_sentinel_rule_details from app.translator.platforms.microsoft.renders.microsoft_sentinel import ( - MicrosoftSentinelFieldValue, + MicrosoftSentinelFieldValueRender, MicrosoftSentinelQueryRender, ) from app.translator.tools.utils import get_rule_description_str @@ -42,7 +42,7 @@ } -class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): +class MicrosoftSentinelRuleFieldValueRender(MicrosoftSentinelFieldValueRender): details: PlatformDetails = microsoft_sentinel_rule_details @@ -50,7 +50,7 @@ class MicrosoftSentinelRuleFieldValue(MicrosoftSentinelFieldValue): class MicrosoftSentinelRuleRender(MicrosoftSentinelQueryRender): details: PlatformDetails = microsoft_sentinel_rule_details or_token = "or" - field_value_map = MicrosoftSentinelRuleFieldValue(or_token=or_token) + field_value_render = MicrosoftSentinelRuleFieldValueRender(or_token=or_token) def __create_mitre_threat(self, meta_info: MetaInfoContainer) -> tuple[list, list]: tactics = set() @@ -92,7 +92,4 @@ def finalize_query( rule["tactics"] = mitre_tactics rule["techniques"] = mitre_techniques json_rule = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return json_rule + rendered_not_supported - return json_rule + return self.wrap_with_not_supported_functions(json_rule, not_supported_functions) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py index 1d2145a7..3298c106 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch.py @@ -24,12 +24,12 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager -from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValue, LuceneQueryRender +from app.translator.platforms.base.lucene.renders.lucene import LuceneFieldValueRender, LuceneQueryRender from app.translator.platforms.opensearch.const import opensearch_query_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -class OpenSearchFieldValue(LuceneFieldValue): +class OpenSearchFieldValueRender(LuceneFieldValueRender): details: PlatformDetails = opensearch_query_details def equal_modifier(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: @@ -102,4 +102,4 @@ class OpenSearchQueryRender(LuceneQueryRender): mappings: OpenSearchMappings = opensearch_mappings or_token = "OR" - field_value_map = OpenSearchFieldValue(or_token=or_token) + field_value_render = OpenSearchFieldValueRender(or_token=or_token) diff --git a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py index 3f68e6c6..106a1fe1 100644 --- a/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py +++ b/uncoder-core/app/translator/platforms/opensearch/renders/opensearch_rule.py @@ -30,13 +30,13 @@ from app.translator.managers import render_manager from app.translator.platforms.opensearch.const import OPENSEARCH_RULE, opensearch_rule_details from app.translator.platforms.opensearch.mapping import OpenSearchMappings, opensearch_mappings -from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValue, OpenSearchQueryRender +from app.translator.platforms.opensearch.renders.opensearch import OpenSearchFieldValueRender, OpenSearchQueryRender _AUTOGENERATED_TEMPLATE = "Autogenerated AWS OpenSearch Rule" _SEVERITIES_MAP = {SeverityType.critical: "5", SeverityType.high: "4", SeverityType.medium: "3", SeverityType.low: "2"} -class OpenSearchRuleFieldValue(OpenSearchFieldValue): +class OpenSearchRuleFieldValueRender(OpenSearchFieldValueRender): details: PlatformDetails = opensearch_rule_details @@ -49,7 +49,7 @@ class OpenSearchRuleRender(OpenSearchQueryRender): and_token = "AND" not_token = "NOT" - field_value_map = OpenSearchRuleFieldValue(or_token=or_token) + field_value_render = OpenSearchRuleFieldValueRender(or_token=or_token) def __init__(self): super().__init__() @@ -76,10 +76,7 @@ def finalize_query( rule["triggers"][0]["severity"] = _SEVERITIES_MAP[meta_info.severity] rule["triggers"][0]["actions"][0]["message_template"]["source"] = str(source).replace(", ", ",\n") rule_str = json.dumps(rule, indent=4, sort_keys=False) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule_str + rendered_not_supported - return rule_str + return self.wrap_with_not_supported_functions(rule_str, not_supported_functions) def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: if isinstance(token, FieldValue): diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index 54f50916..d3300abd 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -20,9 +20,11 @@ from typing import ClassVar, Optional, Union from app.translator.const import DEFAULT_VALUE_TYPE +from app.translator.core.context_vars import preset_log_source_str_ctx_var +from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType from app.translator.core.models.platform_details import PlatformDetails -from app.translator.core.render import BaseQueryFieldValue, PlatformQueryRender +from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue from app.translator.managers import render_manager from app.translator.platforms.palo_alto.const import cortex_xql_query_details @@ -35,7 +37,7 @@ from app.translator.platforms.palo_alto.str_value_manager import cortex_xql_str_value_manager -class CortexXQLFieldValue(BaseQueryFieldValue): +class CortexXQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = cortex_xql_query_details str_value_manager = cortex_xql_str_value_manager @@ -132,6 +134,17 @@ def keywords(self, field: str, value: DEFAULT_VALUE_TYPE) -> str: return f"_raw_log contains {self._pre_process_value(field ,value, value_type=ValueType.value, wrap_str=True)}" +class CortexXQLFieldFieldRender(BaseFieldFieldRender): + operators_map: ClassVar[dict[str, str]] = { + OperatorType.EQ: "=", + OperatorType.NOT_EQ: "!=", + OperatorType.LT: "<", + OperatorType.LTE: "<=", + OperatorType.GT: ">", + OperatorType.GTE: ">=", + } + + @render_manager.register class CortexXQLQueryRender(PlatformQueryRender): details: PlatformDetails = cortex_xql_query_details @@ -149,7 +162,8 @@ class CortexXQLQueryRender(PlatformQueryRender): not_token = "not" query_parts_delimiter = "\n" - field_value_map = CortexXQLFieldValue(or_token=or_token) + field_field_render = CortexXQLFieldFieldRender() + field_value_render = CortexXQLFieldValueRender(or_token=or_token) comment_symbol = "//" is_single_line_comment = False @@ -171,7 +185,8 @@ def process_raw_log_field(self, field: str, field_type: str) -> Optional[str]: def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, functions_prefix: str = "") -> str: functions_prefix = f"{functions_prefix} | " if functions_prefix else "" - return f"{functions_prefix}{log_source_signature}" + log_source_str = preset_log_source_str_ctx_var.get() or str(log_source_signature) + return f"{functions_prefix}{log_source_str}" @staticmethod def _finalize_search_query(query: str) -> str: diff --git a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py index 0f06fb40..cf4a7d51 100644 --- a/uncoder-core/app/translator/platforms/qradar/renders/qradar.py +++ b/uncoder-core/app/translator/platforms/qradar/renders/qradar.py @@ -19,14 +19,15 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.aql.renders.aql import AQLFieldValue, AQLQueryRender +from app.translator.platforms.base.aql.renders.aql import AQLFieldValueRender, AQLQueryRender from app.translator.platforms.qradar.const import qradar_query_details -class QradarFieldValue(AQLFieldValue): +class QradarFieldValueRender(AQLFieldValueRender): details: PlatformDetails = qradar_query_details @render_manager.register class QradarQueryRender(AQLQueryRender): details: PlatformDetails = qradar_query_details + field_value_render = QradarFieldValueRender(or_token=AQLQueryRender.or_token) diff --git a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py index dc33a507..856fd4a3 100644 --- a/uncoder-core/app/translator/platforms/sigma/renders/sigma.py +++ b/uncoder-core/app/translator/platforms/sigma/renders/sigma.py @@ -281,10 +281,10 @@ def __get_source_mapping(self, source_mapping_ids: list[str]) -> SourceMapping: return self.mappings.get_source_mapping(DEFAULT_MAPPING_NAME) - def _generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: + def generate_from_raw_query_container(self, query_container: RawQueryContainer) -> str: raise NotImplementedError - def _generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: + def generate_from_tokenized_query_container(self, query_container: TokenizedQueryContainer) -> str: self.reset_counters() meta_info = query_container.meta_info @@ -316,6 +316,6 @@ def _generate_from_tokenized_query_container(self, query_container: TokenizedQue def generate(self, query_container: Union[RawQueryContainer, TokenizedQueryContainer]) -> str: if isinstance(query_container, RawQueryContainer): - return self._generate_from_raw_query_container(query_container) + return self.generate_from_raw_query_container(query_container) - return self._generate_from_tokenized_query_container(query_container) + return self.generate_from_tokenized_query_container(query_container) diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py index f9404cac..e14c6bfc 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk.py @@ -19,13 +19,13 @@ from app.translator.core.models.platform_details import PlatformDetails from app.translator.managers import render_manager -from app.translator.platforms.base.spl.renders.spl import SplFieldValue, SplQueryRender +from app.translator.platforms.base.spl.renders.spl import SplFieldValueRender, SplQueryRender from app.translator.platforms.splunk.const import splunk_query_details from app.translator.platforms.splunk.functions import SplunkFunctions, splunk_functions from app.translator.platforms.splunk.mapping import SplunkMappings, splunk_mappings -class SplunkFieldValue(SplFieldValue): +class SplunkFieldValueRender(SplFieldValueRender): details: PlatformDetails = splunk_query_details @@ -35,7 +35,7 @@ class SplunkQueryRender(SplQueryRender): or_token = "OR" - field_value_map = SplunkFieldValue(or_token=or_token) + field_value_render = SplunkFieldValueRender(or_token=or_token) mappings: SplunkMappings = splunk_mappings platform_functions: SplunkFunctions = None diff --git a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py index ef0d097d..5dc2096a 100644 --- a/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py +++ b/uncoder-core/app/translator/platforms/splunk/renders/splunk_alert.py @@ -25,14 +25,14 @@ from app.translator.core.models.query_container import MetaInfoContainer from app.translator.managers import render_manager from app.translator.platforms.splunk.const import DEFAULT_SPLUNK_ALERT, splunk_alert_details -from app.translator.platforms.splunk.renders.splunk import SplunkFieldValue, SplunkQueryRender +from app.translator.platforms.splunk.renders.splunk import SplunkFieldValueRender, SplunkQueryRender from app.translator.tools.utils import get_rule_description_str _AUTOGENERATED_TEMPLATE = "Autogenerated Splunk Alert" _SEVERITIES_MAP = {SeverityType.critical: "4", SeverityType.high: "3", SeverityType.medium: "2", SeverityType.low: "1"} -class SplunkAlertFieldValue(SplunkFieldValue): +class SplunkAlertFieldValueRender(SplunkFieldValueRender): details: PlatformDetails = splunk_alert_details @@ -40,7 +40,7 @@ class SplunkAlertFieldValue(SplunkFieldValue): class SplunkAlertRender(SplunkQueryRender): details: PlatformDetails = splunk_alert_details or_token = "OR" - field_value_map = SplunkAlertFieldValue(or_token=or_token) + field_value_render = SplunkAlertFieldValueRender(or_token=or_token) @staticmethod def __create_mitre_threat(meta_info: MetaInfoContainer) -> dict: @@ -74,7 +74,4 @@ def finalize_query( if mitre_techniques: mitre_str = f"action.correlationsearch.annotations = {mitre_techniques})" rule = rule.replace("", mitre_str) - if not_supported_functions: - rendered_not_supported = self.render_not_supported_functions(not_supported_functions) - return rule + rendered_not_supported - return rule + return self.wrap_with_not_supported_functions(rule, not_supported_functions) From fb8f50e4baa0ef41152b7c5cd8ecdac59094c041 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Mon, 24 Jun 2024 13:11:31 +0300 Subject: [PATCH 2/6] fix --- uncoder-core/app/translator/core/render.py | 2 +- .../platforms/logrhythm_axon/renders/logrhythm_axon_query.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index 636f718f..caaaa74d 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -287,7 +287,7 @@ def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) return "".join(result_values) - def wrap_with_meta_info(self, query: str, meta_info: MetaInfoContainer) -> str: + def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str: if wrap_query_with_meta_info_ctx_var.get() and meta_info and (meta_info.id or meta_info.title): meta_info_dict = { "name: ": meta_info.title, diff --git a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py index 0cef42f2..4a288491 100644 --- a/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py +++ b/uncoder-core/app/translator/platforms/logrhythm_axon/renders/logrhythm_axon_query.py @@ -232,12 +232,13 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp f"Uncoder does not support complex regexp for unmapped field:" f" {token.field.source_name} for LogRhythm Axon" ) from exc - return self.group_token % self.logical_operators_map[LogicalOperatorType.OR].join( + joined = self.logical_operators_map[LogicalOperatorType.OR].join( [ self.field_value_render.apply_field_value(field=field, operator=token.operator, value=token.value) for field in mapped_fields ] ) + return self.group_token % joined if len(mapped_fields) > 1 else joined return super().apply_token(token, source_mapping) From 1719f09f1bd4848312afa0eded08d9d46e4e5287 Mon Sep 17 00:00:00 2001 From: "oleksandr.volha" Date: Mon, 24 Jun 2024 13:16:37 +0300 Subject: [PATCH 3/6] add models --- .../translator/core/models/functions/join.py | 26 +++++++++++++++++++ .../translator/core/models/functions/union.py | 12 +++++++++ 2 files changed, 38 insertions(+) create mode 100644 uncoder-core/app/translator/core/models/functions/join.py create mode 100644 uncoder-core/app/translator/core/models/functions/union.py diff --git a/uncoder-core/app/translator/core/models/functions/join.py b/uncoder-core/app/translator/core/models/functions/join.py new file mode 100644 index 00000000..0f44da68 --- /dev/null +++ b/uncoder-core/app/translator/core/models/functions/join.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass, field +from typing import Union + +from app.translator.core.custom_types.functions import FunctionType +from app.translator.core.models.field import Alias, Field +from app.translator.core.models.functions.base import Function +from app.translator.core.models.identifier import Identifier +from app.translator.core.models.query_container import TokenizedQueryContainer +from app.translator.tools.custom_enum import CustomEnum + + +class JoinType(CustomEnum): + inner = "inner" + left = "left" + right = "right" + cross = "cross" + + +@dataclass +class JoinFunction(Function): + name: str = FunctionType.join + alias: Alias = None + type_: str = JoinType.inner + tokenized_query_container: TokenizedQueryContainer = None + condition: list[Union[Alias, Field, Identifier]] = field(default_factory=list) + preset_log_source_str: str = None diff --git a/uncoder-core/app/translator/core/models/functions/union.py b/uncoder-core/app/translator/core/models/functions/union.py new file mode 100644 index 00000000..ef4d4179 --- /dev/null +++ b/uncoder-core/app/translator/core/models/functions/union.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass + +from app.translator.core.custom_types.functions import FunctionType +from app.translator.core.models.functions.base import Function +from app.translator.core.models.query_container import TokenizedQueryContainer + + +@dataclass +class UnionFunction(Function): + name: str = FunctionType.union + tokenized_query_container: TokenizedQueryContainer = None + preset_log_source_str: str = None From 93ceff9996799e093a1883534657b1232c134fd9 Mon Sep 17 00:00:00 2001 From: "oleksandr.volha" Date: Mon, 24 Jun 2024 13:19:21 +0300 Subject: [PATCH 4/6] add name --- uncoder-core/app/translator/core/custom_types/functions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/uncoder-core/app/translator/core/custom_types/functions.py b/uncoder-core/app/translator/core/custom_types/functions.py index bb31e77f..8e609a03 100644 --- a/uncoder-core/app/translator/core/custom_types/functions.py +++ b/uncoder-core/app/translator/core/custom_types/functions.py @@ -28,6 +28,7 @@ class FunctionType(CustomEnum): bin = "bin" eval = "eval" fields = "fields" + join = "join" rename = "rename" search = "search" sort_limit = "sort_limit" From eccbae370774bcfd3976e40a7421b818b9d92148 Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Wed, 26 Jun 2024 14:30:02 +0300 Subject: [PATCH 5/6] resolve conflicts --- .../app/translator/core/custom_types/time.py | 14 ++++++++++ .../app/translator/core/exceptions/core.py | 18 +++++++++++-- .../app/translator/core/models/field.py | 5 ++++ uncoder-core/app/translator/core/render.py | 8 +++++- .../platforms/palo_alto_cortex/default.yml | 1 + .../platforms/palo_alto_cortex/dns.yml | 3 ++- .../windows_registry_event.yml | 3 ++- .../mappings/platforms/qradar/default.yml | 17 +++++++++--- .../mappings/platforms/qradar/dns.yml | 3 ++- .../mappings/platforms/qradar/proxy.yml | 1 + .../platforms/qradar/windows_security.yml | 5 +++- .../platforms/splunk/aws_cloudtrail.yml | 4 +-- .../mappings/platforms/splunk/aws_eks.yml | 4 +-- .../splunk/azure_AzureDiagnostics.yml | 4 +-- .../splunk/azure_BehaviorAnalytics.yml | 4 +-- .../azure_aadnoninteractiveusersigninlogs.yml | 4 +-- .../platforms/splunk/azure_azureactivity.yml | 4 +-- .../platforms/splunk/azure_azuread.yml | 4 +-- .../platforms/splunk/azure_signinlogs.yml | 4 +-- .../mappings/platforms/splunk/firewall.yml | 4 +-- .../platforms/splunk/gcp_gcp.audit.yml | 2 +- .../mappings/platforms/splunk/gcp_pubsub.yml | 2 +- .../platforms/splunk/linux_auditd.yml | 4 +-- .../mappings/platforms/splunk/okta_okta.yml | 4 +-- .../platforms/splunk/windows_bits_client.yml | 4 +-- .../platforms/splunk/windows_dns_query.yml | 4 +-- .../platforms/splunk/windows_driver_load.yml | 4 +-- .../platforms/splunk/windows_file_access.yml | 4 +-- .../platforms/splunk/windows_file_change.yml | 4 +-- .../platforms/splunk/windows_file_create.yml | 4 +-- .../platforms/splunk/windows_file_delete.yml | 4 +-- .../platforms/splunk/windows_file_event.yml | 4 +-- .../platforms/splunk/windows_file_rename.yml | 4 +-- .../platforms/splunk/windows_image_load.yml | 4 +-- .../platforms/splunk/windows_ldap_debug.yml | 4 +-- .../splunk/windows_network_connection.yml | 4 +-- .../platforms/splunk/windows_ntlm.yml | 4 +-- .../splunk/windows_registry_event.yml | 4 +-- .../platforms/splunk/windows_sysmon.yml | 4 +-- .../platforms/splunk/windows_wmi_event.yml | 4 +-- .../palo_alto/renders/cortex_xsiam.py | 26 +++++++++++++++++++ .../app/translator/platforms/sigma/mapping.py | 6 ++--- .../translator/platforms/splunk/mapping.py | 4 +-- 43 files changed, 155 insertions(+), 71 deletions(-) diff --git a/uncoder-core/app/translator/core/custom_types/time.py b/uncoder-core/app/translator/core/custom_types/time.py index 1d5f15b8..4cdc71fe 100644 --- a/uncoder-core/app/translator/core/custom_types/time.py +++ b/uncoder-core/app/translator/core/custom_types/time.py @@ -7,3 +7,17 @@ class TimeFrameType(CustomEnum): days = "days" hours = "hours" minutes = "minutes" + + +class TimePartType(CustomEnum): + day = "day" + day_of_week = "day_of_week" + day_of_year = "day_of_year" + hour = "hour" + microsecond = "microsecond" + millisecond = "millisecond" + minute = "minute" + month = "month" + quarter = "quarter" + second = "second" + year = "year" diff --git a/uncoder-core/app/translator/core/exceptions/core.py b/uncoder-core/app/translator/core/exceptions/core.py index 68c66962..47810576 100644 --- a/uncoder-core/app/translator/core/exceptions/core.py +++ b/uncoder-core/app/translator/core/exceptions/core.py @@ -1,3 +1,6 @@ +from typing import Optional + + class NotImplementedException(BaseException): ... @@ -7,8 +10,19 @@ class BasePlatformException(BaseException): class StrictPlatformException(BasePlatformException): - def __init__(self, platform_name: str, field_name: str): - message = f"Platform {platform_name} has strict mapping. Source field {field_name} has no mapping." + field_name: str = None + + def __init__( + self, platform_name: str, field_name: str, mapping: Optional[str] = None, detected_fields: Optional[list] = None + ): + message = ( + f"Platform {platform_name} has strict mapping. " + f"Source fields: {', '.join(detected_fields) if detected_fields else field_name} has no mapping." + f" Mapping file: {mapping}." + if mapping + else "" + ) + self.field_name = field_name super().__init__(message) diff --git a/uncoder-core/app/translator/core/models/field.py b/uncoder-core/app/translator/core/models/field.py index 0c430382..95556dcc 100644 --- a/uncoder-core/app/translator/core/models/field.py +++ b/uncoder-core/app/translator/core/models/field.py @@ -76,6 +76,11 @@ def value(self) -> Union[int, str, StrValue, list[Union[int, str, StrValue]]]: return self.values[0] return self.values + @value.setter + def value(self, new_value: Union[int, str, StrValue, list[Union[int, str, StrValue]]]) -> None: + self.values = [] + self.__add_value(new_value) + def __add_value(self, value: Optional[Union[int, str, StrValue, list, tuple]]) -> None: if value and isinstance(value, (list, tuple)): for v in value: diff --git a/uncoder-core/app/translator/core/render.py b/uncoder-core/app/translator/core/render.py index caaaa74d..b66f4430 100644 --- a/uncoder-core/app/translator/core/render.py +++ b/uncoder-core/app/translator/core/render.py @@ -283,8 +283,14 @@ def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapp def generate_query(self, tokens: list[TOKEN_TYPE], source_mapping: SourceMapping) -> str: result_values = [] + unmapped_fields = set() for token in tokens: - result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) + try: + result_values.append(self.apply_token(token=token, source_mapping=source_mapping)) + except StrictPlatformException as err: + unmapped_fields.add(err.field_name) + if unmapped_fields: + raise StrictPlatformException(self.details.name, "", source_mapping.source_id, sorted(unmapped_fields)) return "".join(result_values) def wrap_with_meta_info(self, query: str, meta_info: Optional[MetaInfoContainer]) -> str: diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml index f6b25023..fa904aaf 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/default.yml @@ -125,3 +125,4 @@ field_mapping: SourceOS: xdm.source.host.os DestinationOS: xdm.target.host.os url_category: xdm.network.http.url_category + EventSeverity: xdm.alert.severity diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml index e489fd50..e279a60a 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/dns.yml @@ -10,4 +10,5 @@ field_mapping: #dns-record: dns-record dns_query_name: xdm.network.dns.dns_question.name QueryName: xdm.network.dns.dns_question.name - query: xdm.network.dns.dns_question.name \ No newline at end of file + query: xdm.network.dns.dns_question.name + dns-record-type: xdm.network.dns.dns_question.type \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml index 86110049..04abb36b 100644 --- a/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/palo_alto_cortex/windows_registry_event.yml @@ -28,4 +28,5 @@ field_mapping: ParentIntegrityLevel: causality_actor_process_integrity_level ParentLogonId: causality_actor_process_logon_id ParentProduct: causality_actor_process_signature_product - ParentCompany: causality_actor_process_signature_vendor \ No newline at end of file + ParentCompany: causality_actor_process_signature_vendor + EventType: event_sub_type \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml index 6e798034..004e10c7 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/default.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/default.yml @@ -13,9 +13,12 @@ field_mapping: dst-port: - DstPort - DestinationPort + - remoteport dst-hostname: DstHost src-hostname: SrcHost - src-port: SourcePort + src-port: + - SourcePort + - localport src-ip: - sourceip - source_ip @@ -27,11 +30,14 @@ field_mapping: - destination_ip - destinationIP - destinationaddress + - destination User: - userName - EventUserName CommandLine: Command - Protocol: IPProtocol + Protocol: + - IPProtocol + - protocol Application: - Application - application @@ -57,6 +63,7 @@ field_mapping: SourceMAC: - SourceMAC - MAC + - sourceMAC DestinationMAC: DestinationMAC SourceOS: - SourceOS @@ -64,4 +71,8 @@ field_mapping: DestinationOS: DestinationOS TargetUserName: DestinationUserName SourceUserName: SourceUserName - url_category: XForceCategoryByURL \ No newline at end of file + url_category: XForceCategoryByURL + EventSeverity: EventSeverity + Source: + - Source + - source \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml b/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml index 048a4bd3..d9aad78e 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/dns.yml @@ -12,4 +12,5 @@ field_mapping: dns-query: URL parent-domain: parent-domain dns-answer: dns-answer - dns-record: URL \ No newline at end of file + dns-record: URL + dns-record-type: DNSRecordType \ No newline at end of file diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml index 58393ac0..193bc79c 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/proxy.yml @@ -24,6 +24,7 @@ field_mapping: cs-host: - UrlHost - URL Host + - URL Domain cs-referrer: - URL Referrer - Referrer URL diff --git a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml index 7d01b97e..53b37952 100644 --- a/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml +++ b/uncoder-core/app/translator/mappings/platforms/qradar/windows_security.yml @@ -41,7 +41,9 @@ field_mapping: LinkName: LinkName MemberName: MemberName MemberSid: MemberSid - NewProcessName: Process Name + NewProcessName: + - Process Name + - New Process Name ObjectClass: ObjectClass ObjectName: - Object Name @@ -122,6 +124,7 @@ field_mapping: ServiceFileName: - Service Filename - ServiceFileName + - Service File Name SecurityDescriptor: SecurityDescriptor ServiceName: Service Name ShareName: diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml b/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml index acd62dbc..96bb06b8 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/aws_cloudtrail.yml @@ -3,10 +3,10 @@ source: aws_cloudtrail log_source: - source_type: [aws:cloudtrail] + sourcetype: [aws:cloudtrail] default_log_source: - source_type: aws:cloudtrail + sourcetype: aws:cloudtrail field_mapping: eventSource: eventSource diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml b/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml index 32302e30..38e225d7 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/aws_eks.yml @@ -3,10 +3,10 @@ source: aws_eks log_source: - source_type: [aws:*] + sourcetype: [aws:*] default_log_source: - source_type: aws:* + sourcetype: aws:* field_mapping: annotations.authorization.k8s.io\/decision: annotations.authorization.k8s.io\/decision diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml index 5cff60da..90fd75a1 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_AzureDiagnostics.yml @@ -3,10 +3,10 @@ source: azure_AzureDiagnostics log_source: - source_type: [azure:*] + sourcetype: [azure:*] default_log_source: - source_type: azure:* + sourcetype: azure:* field_mapping: ResultDescription: ResultDescription diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml index 379004da..e1f17620 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_BehaviorAnalytics.yml @@ -3,10 +3,10 @@ source: azure_BehaviorAnalytics log_source: - source_type: [azure:*] + sourcetype: [azure:*] default_log_source: - source_type: azure:* + sourcetype: azure:* field_mapping: ActionType: ActionType diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml index 3e994fc5..ad6bb5eb 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_aadnoninteractiveusersigninlogs.yml @@ -3,10 +3,10 @@ source: azure_aadnoninteractiveusersigninlogs log_source: - source_type: [azure:*] + sourcetype: [azure:*] default_log_source: - source_type: azure:* + sourcetype: azure:* field_mapping: UserAgent: UserAgent diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml index d3623983..337125f4 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azureactivity.yml @@ -3,10 +3,10 @@ source: azure_azureactivity log_source: - source_type: [mscs:azure:*, azure:*] + sourcetype: [mscs:azure:*, azure:*] default_log_source: - source_type: mscs:azure:* + sourcetype: mscs:azure:* field_mapping: ActivityStatus: ActivityStatus diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml index 5f393c91..69e3d195 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_azuread.yml @@ -3,10 +3,10 @@ source: azure_azuread log_source: - source_type: [azure:aad:*] + sourcetype: [azure:aad:*] default_log_source: - source_type: azure:aad:* + sourcetype: azure:aad:* field_mapping: ActivityDisplayName: ActivityDisplayName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml b/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml index 23b7569b..4f669d89 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/azure_signinlogs.yml @@ -3,10 +3,10 @@ source: azure_signinlogs log_source: - source_type: [azure:aad:*] + sourcetype: [azure:aad:*] default_log_source: - source_type: azure:aad:* + sourcetype: azure:aad:* field_mapping: AppDisplayName: AppDisplayName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml b/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml index f40ef682..ed886d9c 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/firewall.yml @@ -3,11 +3,11 @@ source: firewall log_source: - source_type: [fortigate_traffic] + sourcetype: [fortigate_traffic] index: [fortigate] default_log_source: - source_type: fortigate_traffic + sourcetype: fortigate_traffic index: fortigate field_mapping: diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml index ef92fb58..be54b882 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_gcp.audit.yml @@ -3,7 +3,7 @@ source: gcp_gcp.audit log_source: - source_type: [google:gcp:*] + sourcetype: [google:gcp:*] default_log_source: index: google:gcp:* diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml index 7ab8483c..dbfd2736 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/gcp_pubsub.yml @@ -3,7 +3,7 @@ source: gcp_pubsub log_source: - source_type: [google:gcp:*] + sourcetype: [google:gcp:*] default_log_source: index: google:gcp:* diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml b/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml index ee3ac161..afd115b0 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/linux_auditd.yml @@ -3,10 +3,10 @@ source: linux_auditd log_source: - source_type: [linux:audit] + sourcetype: [linux:audit] default_log_source: - source_type: linux:audit + sourcetype: linux:audit field_mapping: a0: a0 diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml b/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml index 3ee6d0e1..3f55621f 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/okta_okta.yml @@ -3,10 +3,10 @@ source: okta_okta log_source: - source_type: [OktaIM2:*] + sourcetype: [OktaIM2:*] default_log_source: - source_type: OktaIM2:* + sourcetype: OktaIM2:* field_mapping: client.user.id: client.user.id diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml index 014287eb..babbd610 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_bits_client.yml @@ -2,10 +2,10 @@ platform: Splunk source: windows_bits_client log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Bits-Client/Operational field_mapping: LocalName: LocalName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml index 698e62cc..d8e40100 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_dns_query.yml @@ -4,11 +4,11 @@ source: windows_dns_query log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: Image: Image diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml index f8248b8e..86b76510 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_driver_load.yml @@ -4,11 +4,11 @@ source: windows_driver_load log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: ImageLoaded: ImageLoaded diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml index 5c1c64f2..48ab5786 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_access.yml @@ -4,11 +4,11 @@ source: windows_file_access log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml index 0114b7e0..f45393aa 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_change.yml @@ -4,11 +4,11 @@ source: windows_file_change log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml index d9b0d8c0..485ea463 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_create.yml @@ -4,11 +4,11 @@ source: windows_file_create log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml index 8b82cc38..13660235 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_delete.yml @@ -4,11 +4,11 @@ source: windows_file_delete log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml index 278b9b30..ed0855d3 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_event.yml @@ -4,11 +4,11 @@ source: windows_file_event log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml index 10390535..dae50085 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_file_rename.yml @@ -4,11 +4,11 @@ source: windows_file_rename log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CreationUtcTime: CreationUtcTime diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml index 8f427639..3cc22f55 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_image_load.yml @@ -4,11 +4,11 @@ source: windows_image_load log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: Image: Image diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml index 8fc85d34..f8241ba4 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ldap_debug.yml @@ -3,10 +3,10 @@ source: windows_ldap_debug log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug] + sourcetype: [XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug + sourcetype: XmlWinEventLog:Microsoft-Windows-LDAP-Client/Debug field_mapping: EventID: EventID diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml index d8260810..7a92b32c 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_network_connection.yml @@ -4,11 +4,11 @@ source: windows_network_connection log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: Image: Image diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml index 3ea2c8ea..7902c0fe 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_ntlm.yml @@ -3,10 +3,10 @@ source: windows_ntlm log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-NTLM/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-NTLM/Operational] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-NTLM/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-NTLM/Operational field_mapping: WorkstationName: WorkstationName diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml index 8cbe38f3..a2169567 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_registry_event.yml @@ -4,11 +4,11 @@ source: windows_registry_event log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: TargetObject: TargetObject diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml index a361471a..89bf98e0 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_sysmon.yml @@ -3,11 +3,11 @@ source: windows_sysmon log_source: source: [WinEventLog:Microsoft-Windows-Sysmon/Operational] - source_type: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-Sysmon/Operational] default_log_source: source: WinEventLog:Microsoft-Windows-Sysmon/Operational - source_type: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-Sysmon/Operational field_mapping: CommandLine: CommandLine diff --git a/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml b/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml index 5e1e47bd..b1e415d0 100644 --- a/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml +++ b/uncoder-core/app/translator/mappings/platforms/splunk/windows_wmi_event.yml @@ -3,10 +3,10 @@ source: windows_wmi_event log_source: - source_type: [XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational] + sourcetype: [XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational] default_log_source: - source_type: XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational + sourcetype: XmlWinEventLog:Microsoft-Windows-WMI-Activity/Operational field_mapping: Destination: Destination diff --git a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py index d3300abd..1f2f7d04 100644 --- a/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py +++ b/uncoder-core/app/translator/platforms/palo_alto/renders/cortex_xsiam.py @@ -23,6 +23,9 @@ from app.translator.core.context_vars import preset_log_source_str_ctx_var from app.translator.core.custom_types.tokens import OperatorType from app.translator.core.custom_types.values import ValueType +from app.translator.core.mapping import SourceMapping +from app.translator.core.models.field import FieldValue, Keyword +from app.translator.core.models.identifier import Identifier from app.translator.core.models.platform_details import PlatformDetails from app.translator.core.render import BaseFieldFieldRender, BaseFieldValueRender, PlatformQueryRender from app.translator.core.str_value_manager import StrValue @@ -36,6 +39,16 @@ ) from app.translator.platforms.palo_alto.str_value_manager import cortex_xql_str_value_manager +SOURCE_MAPPING_TO_FIELD_VALUE_MAP = { + "windows_registry_event": { + "EventType": { + "SetValue": "REGISTRY_SET_VALUE", + "DeleteValue": "REGISTRY_DELETE_VALUE", + "CreateKey": "REGISTRY_CREATE_KEY", + } + } +} + class CortexXQLFieldValueRender(BaseFieldValueRender): details: PlatformDetails = cortex_xql_query_details @@ -188,6 +201,19 @@ def generate_prefix(self, log_source_signature: CortexXQLLogSourceSignature, fun log_source_str = preset_log_source_str_ctx_var.get() or str(log_source_signature) return f"{functions_prefix}{log_source_str}" + def apply_token(self, token: Union[FieldValue, Keyword, Identifier], source_mapping: SourceMapping) -> str: + if isinstance(token, FieldValue): + field_name = token.field.source_name + if values_map := SOURCE_MAPPING_TO_FIELD_VALUE_MAP.get(source_mapping.source_id, {}).get(field_name): + values_to_update = [] + for token_value in token.values: + mapped_value: str = values_map.get(token_value, token_value) + values_to_update.append( + StrValue(value=mapped_value, split_value=mapped_value.split()) if mapped_value else token_value + ) + token.value = values_to_update + return super().apply_token(token=token, source_mapping=source_mapping) + @staticmethod def _finalize_search_query(query: str) -> str: return f"| filter {query}" if query else "" diff --git a/uncoder-core/app/translator/platforms/sigma/mapping.py b/uncoder-core/app/translator/platforms/sigma/mapping.py index 3f23700d..1af791ac 100644 --- a/uncoder-core/app/translator/platforms/sigma/mapping.py +++ b/uncoder-core/app/translator/platforms/sigma/mapping.py @@ -19,9 +19,9 @@ def __init__( def is_suitable( self, service: Optional[list[str]], product: Optional[list[str]], category: Optional[list[str]] ) -> bool: - product_match = set(product or []).issubset(self.products) if product else False - category_match = set(category or []).issubset(self.categories) if category else False - service_match = set(service or []).issubset(self.services) if service else False + product_match = set(product_.lower() for product_ in product or []).issubset(self.products) if product else False + category_match = set(category_.lower() for category_ in category or []).issubset(self.categories) if category else False + service_match = set(service_.lower() for service_ in service or [] or []).issubset(self.services) if service else False if not product and not service: return category_match return product_match and service_match or product_match and category_match diff --git a/uncoder-core/app/translator/platforms/splunk/mapping.py b/uncoder-core/app/translator/platforms/splunk/mapping.py index 2f9c4a8d..1851b8af 100644 --- a/uncoder-core/app/translator/platforms/splunk/mapping.py +++ b/uncoder-core/app/translator/platforms/splunk/mapping.py @@ -42,8 +42,8 @@ def prepare_log_source_signature(self, mapping: dict) -> SplunkLogSourceSignatur default_log_source = mapping["default_log_source"] return SplunkLogSourceSignature( sources=log_source.get("source"), - source_types=log_source.get("source_type"), - source_categories=log_source.get("source_category"), + source_types=log_source.get("sourcetype"), + source_categories=log_source.get("sourcecategory"), indices=log_source.get("index"), default_source=default_log_source, ) From 8d954b3b7599af76a35bebc3553fdd79583a5b2e Mon Sep 17 00:00:00 2001 From: Oleksandr Volha Date: Thu, 27 Jun 2024 10:27:40 +0300 Subject: [PATCH 6/6] fix --- uncoder-core/app/translator/core/tokenizer.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/uncoder-core/app/translator/core/tokenizer.py b/uncoder-core/app/translator/core/tokenizer.py index 2ecbc39c..7530af5f 100644 --- a/uncoder-core/app/translator/core/tokenizer.py +++ b/uncoder-core/app/translator/core/tokenizer.py @@ -344,11 +344,7 @@ def get_field_tokens_from_func_args( # noqa: PLR0912 result.extend(self.get_field_tokens_from_func_args(args=arg.by_clauses)) result.extend(self.get_field_tokens_from_func_args(args=[arg.filter_])) elif isinstance(arg, (JoinFunction, UnionFunction)): - result.extend(self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.tokens)) - result.extend( - self.get_field_tokens_from_func_args(args=arg.tokenized_query_container.functions.functions) - ) - result.extend(self.get_field_tokens_from_func_args(args=arg.condition)) + continue elif isinstance(arg, Function): result.extend(self.get_field_tokens_from_func_args(args=arg.args)) elif isinstance(arg, SortArg) and isinstance(arg.field, Field):