Merge branch 'elastic:DAC-feature' into DAC-feature

.github/workflows/backport.yml

@@ -98,6 +98,7 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
+          pip cache purge
           pip install .[dev]
 
       - name: Prune non-${{matrix.target_branch}} rules

.github/workflows/lock-versions.yml

@@ -70,7 +70,7 @@ jobs:
           labels: "backport: auto"
 
       - name: Archive production artifacts
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
           name: release-files
           path: |

.github/workflows/pythonpackage.yml

@@ -14,8 +14,14 @@ jobs:
       CUSTOM_RULES_DIR: ${{ secrets.CUSTOM_RULES_DIR }}
 
     steps:
-    - uses: actions/checkout@v2
-
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+
+    - name: Fetch main branch
+      run: |
+        git fetch origin main:refs/remotes/origin/main
+ 
     - name: Set up Python 3.12
       uses: actions/setup-python@v5
       with:
@@ -47,7 +53,7 @@ jobs:
         python -m detection_rules dev build-release $GENERATE_NAVIGATOR_FILES
 
     - name: Archive production artifacts for branch builds
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       if: |
         github.event_name == 'push'
       with:

.github/workflows/release-fleet.yml

@@ -149,7 +149,7 @@ jobs:
               $DRAFT_ARGS
 
         - name: Archive production artifacts
-          uses: actions/upload-artifact@v3
+          uses: actions/upload-artifact@v4
           with:
             name: release-files
             path: |

CLI.md

@@ -475,6 +475,8 @@ python -m detection_rules kibana import-rules -d test-export-rules -o
 
 ### Exporting rules
 
+This command should be run with the `CUSTOM_RULES_DIR` envvar set, that way proper validation is applied to versioning when the rules are downloaded. See the [custom rules docs](docs/custom-rules.md) for more information.
+
 Example of a rule exporting, with errors skipped
 
 ```

detection_rules/__init__.py

@@ -11,6 +11,7 @@
 assert (3, 12) <= sys.version_info < (4, 0), "Only Python 3.12+ supported"
 
 from . import (  # noqa: E402
+    custom_schemas,
     custom_rules,
     devtools,
     docs,
@@ -30,6 +31,7 @@
 
 __all__ = (
     'custom_rules',
+    'custom_schemas',
     'devtools',
     'docs',
     'eswrap',

detection_rules/__main__.py

@@ -5,18 +5,19 @@
 
 # coding=utf-8
 """Shell for detection-rules."""
-import os
 import sys
+from pathlib import Path
 
 import click
 
 assert (3, 12) <= sys.version_info < (4, 0), "Only Python 3.12+ supported"
 
+
 from .main import root  # noqa: E402
 
-CURR_DIR = os.path.dirname(os.path.abspath(__file__))
-CLI_DIR = os.path.dirname(CURR_DIR)
-ROOT_DIR = os.path.dirname(CLI_DIR)
+CURR_DIR = Path(__file__).resolve().parent
+CLI_DIR = CURR_DIR.parent
+ROOT_DIR = CLI_DIR.parent
 
 BANNER = r"""
 █▀▀▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄▄▄ ▄   ▄      █▀▀▄ ▄  ▄ ▄   ▄▄▄ ▄▄▄

detection_rules/attack.py

@@ -17,8 +17,8 @@
 from .utils import cached, clear_caches, get_etc_path, get_etc_glob_path, read_gzip, gzip_compress
 
 PLATFORMS = ['Windows', 'macOS', 'Linux']
-CROSSWALK_FILE = Path(get_etc_path('attack-crosswalk.json'))
-TECHNIQUES_REDIRECT_FILE = Path(get_etc_path('attack-technique-redirects.json'))
+CROSSWALK_FILE = get_etc_path('attack-crosswalk.json')
+TECHNIQUES_REDIRECT_FILE = get_etc_path('attack-technique-redirects.json')
 
 tactics_map = {}
 
@@ -28,17 +28,17 @@ def load_techniques_redirect() -> dict:
     return json.loads(TECHNIQUES_REDIRECT_FILE.read_text())['mapping']
 
 
-def get_attack_file_path() -> str:
+def get_attack_file_path() -> Path:
     pattern = 'attack-v*.json.gz'
     attack_file = get_etc_glob_path(pattern)
     if len(attack_file) < 1:
         raise FileNotFoundError(f'Missing required {pattern} file')
     elif len(attack_file) != 1:
         raise FileExistsError(f'Multiple files found with {pattern} pattern. Only one is allowed')
-    return attack_file[0]
+    return Path(attack_file[0])
 
 
-_, _attack_path_base = get_attack_file_path().split('-v')
+_, _attack_path_base = str(get_attack_file_path()).split('-v')
 _ext_length = len('.json.gz')
 CURRENT_ATTACK_VERSION = _attack_path_base[:-_ext_length]
 
@@ -98,7 +98,7 @@ def load_attack_gz() -> dict:
 
 def refresh_attack_data(save=True) -> (Optional[dict], Optional[bytes]):
     """Refresh ATT&CK data from Mitre."""
-    attack_path = Path(get_attack_file_path())
+    attack_path = get_attack_file_path()
     filename, _, _ = attack_path.name.rsplit('.', 2)
 
     def get_version_from_tag(name, pattern='att&ck-v'):
@@ -126,7 +126,7 @@ def get_version_from_tag(name, pattern='att&ck-v'):
     compressed = gzip_compress(json.dumps(attack_data, sort_keys=True))
 
     if save:
-        new_path = Path(get_etc_path(f'attack-v{latest_version}.json.gz'))
+        new_path = get_etc_path(f'attack-v{latest_version}.json.gz')
         new_path.write_bytes(compressed)
         attack_path.unlink()
         print(f'Replaced file: {attack_path} with {new_path}')

detection_rules/cli_utils.py

@@ -5,6 +5,7 @@
 
 import copy
 import datetime
+import functools
 import os
 import typing
 from pathlib import Path
@@ -13,13 +14,12 @@
 import click
 
 import kql
-import functools
+
 from . import ecs
-from .attack import matrix, tactics, build_threat_map_entry
-from .rule import TOMLRule, TOMLRuleContents
-from .rule_loader import (RuleCollection,
-                          DEFAULT_PREBUILT_RULES_DIRS,
-                          DEFAULT_PREBUILT_BBR_DIRS,
+from .attack import build_threat_map_entry, matrix, tactics
+from .rule import BYPASS_VERSION_LOCK, TOMLRule, TOMLRuleContents
+from .rule_loader import (DEFAULT_PREBUILT_BBR_DIRS,
+                          DEFAULT_PREBUILT_RULES_DIRS, RuleCollection,
                           dict_filter)
 from .schemas import definitions
 from .utils import clear_caches
@@ -132,8 +132,8 @@ def rule_prompt(path=None, rule_type=None, required_only=True, save=True, verbos
             contents[name] = rule_type
             continue
 
-        # these are set at package release time
-        if name == 'version':
+        # these are set at package release time depending on the version strategy
+        if (name == 'version' or name == 'revision') and not BYPASS_VERSION_LOCK:
             continue
 
         if required_only and name not in required_fields:

detection_rules/config.py

@@ -187,6 +187,7 @@ class RulesConfig:
 
     action_dir: Optional[Path] = None
     bbr_rules_dirs: Optional[List[Path]] = field(default_factory=list)
+    bypass_version_lock: bool = False
     exception_dir: Optional[Path] = None
 
     def __post_init__(self):
@@ -262,6 +263,9 @@ def parse_rules_config(path: Optional[Path] = None) -> RulesConfig:
     # paths are relative
     contents['rule_dirs'] = [base_dir.joinpath(d).resolve() for d in loaded.get('rule_dirs')]
 
+    # version strategy
+    contents['bypass_version_lock'] = loaded.get('bypass_version_lock', False)
+
     # bbr_rules_dirs
     # paths are relative
     if loaded.get('bbr_rules_dirs'):

detection_rules/custom_schemas.py

@@ -0,0 +1,35 @@
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0; you may not use this file except in compliance with the Elastic License
+# 2.0.
+
+"""Custom Schemas management."""
+from pathlib import Path
+
+import eql
+import eql.types
+
+from .config import parse_rules_config
+from .utils import cached
+
+RULES_CONFIG = parse_rules_config()
+RESERVED_SCHEMA_NAMES = ["beats", "ecs", "endgame"]
+
+
+@cached
+def get_custom_schemas(stack_version: str) -> dict:
+    """Load custom schemas if present."""
+    custom_schema_dump = {}
+    stack_schema_map = RULES_CONFIG.stack_schema_map[stack_version]
+
+    for schema, value in stack_schema_map.items():
+        if schema not in RESERVED_SCHEMA_NAMES:
+            schema_path = Path(value)
+            if not schema_path.is_absolute():
+                schema_path = RULES_CONFIG.stack_schema_map_file.parent / value
+            if schema_path.is_file():
+                custom_schema_dump.update(eql.utils.load_dump(str(schema_path)))
+            else:
+                raise ValueError(f"Custom schema must be a file: {schema_path}")
+
+    return custom_schema_dump

detection_rules/devtools.py

@@ -85,10 +85,21 @@ def dev_group():
 @click.option('--generate-navigator', is_flag=True, help='Generate ATT&CK navigator files')
 @click.option('--generate-docs', is_flag=True, default=False, help='Generate markdown documentation')
 @click.option('--update-message', type=str, help='Update message for new package')
-def build_release(config_file, update_version_lock: bool, generate_navigator: bool, generate_docs: str,
-                  update_message: str, release=None, verbose=True):
+@click.pass_context
+def build_release(ctx: click.Context, config_file, update_version_lock: bool, generate_navigator: bool,
+                  generate_docs: str, update_message: str, release=None, verbose=True):
     """Assemble all the rules into Kibana-ready release files."""
+    if RULES_CONFIG.bypass_version_lock:
+        click.echo('WARNING: You cannot run this command when the versioning strategy is configured to bypass the '
+                   'version lock. Set `bypass_version_lock` to `False` in the rules config to use the version lock.')
+        ctx.exit()
+
     config = load_dump(config_file)['package']
+
+    err_msg = f'No `registry_data` in package config. Please see the {get_etc_path("package.yaml")} file for an' \
+              f' example on how to supply this field in {PACKAGE_FILE}.'
+    assert 'registry_data' in config, err_msg
+
     registry_data = config['registry_data']
 
     if generate_navigator:
@@ -104,6 +115,7 @@ def build_release(config_file, update_version_lock: bool, generate_navigator: bo
 
     if update_version_lock:
         loaded_version_lock.manage_versions(package.rules, save_changes=True, verbose=verbose)
+
     package.save(verbose=verbose)
 
     previous_pkg_version = find_latest_integration_version("security_detection_engine", "ga",
@@ -335,8 +347,9 @@ def prune_staging_area(target_stack_version: str, dry_run: bool, exception_list:
 
 @dev_group.command('update-lock-versions')
 @click.argument('rule-ids', nargs=-1, required=False)
+@click.pass_context
 @click.option('--force', is_flag=True, help='Force update without confirmation')
-def update_lock_versions(rule_ids: Tuple[str, ...], force: bool):
+def update_lock_versions(ctx: click.Context, rule_ids: Tuple[str, ...], force: bool):
     """Update rule hashes in version.lock.json file without bumping version."""
     rules = RuleCollection.default()
 
@@ -350,6 +363,11 @@ def update_lock_versions(rule_ids: Tuple[str, ...], force: bool):
     ):
         return
 
+    if RULES_CONFIG.bypass_version_lock:
+        click.echo('WARNING: You cannot run this command when the versioning strategy is configured to bypass the '
+                   'version lock. Set `bypass_version_lock` to `False` in the rules config to use the version lock.')
+        ctx.exit()
+
     # this command may not function as expected anymore due to previous changes eliminating the use of add_new=False
     changed, new, _ = loaded_version_lock.manage_versions(rules, exclude_version_update=True, save_changes=True)
 
@@ -442,7 +460,7 @@ def integrations_pr(ctx: click.Context, local_repo: str, token: str, draft: bool
     stack_version = Package.load_configs()["name"]
     package_version = Package.load_configs()["registry_data"]["version"]
 
-    release_dir = Path(RELEASE_DIR) / stack_version / "fleet" / package_version
+    release_dir = RELEASE_DIR / stack_version / "fleet" / package_version
     message = f"[Security Rules] Update security rules package to v{package_version}"
 
     if not release_dir.exists():
@@ -582,7 +600,7 @@ def license_check(ctx, ignore_directory):
     """Check that all code files contain a valid license."""
     ignore_directory += ("env",)
     failed = False
-    base_path = Path(get_path())
+    base_path = get_path()
 
     for path in base_path.rglob('*.py'):
         relative_path = path.relative_to(base_path)
@@ -625,7 +643,7 @@ def test_version_lock(ctx: click.Context, branches: tuple, remote: str):
     finally:
         rules_config = ctx.obj['rules_config']
         diff = git('--no-pager', 'diff', str(rules_config.version_lock_file))
-        outfile = Path(get_path()).joinpath('lock-diff.txt')
+        outfile = get_path() / 'lock-diff.txt'
         outfile.write_text(diff)
         click.echo(f'diff saved to {outfile}')
 
@@ -721,29 +739,32 @@ def add_github_meta(this_rule: TOMLRule, status: str, original_rule_id: Optional
 
 @dev_group.command('deprecate-rule')
 @click.argument('rule-file', type=Path)
+@click.option('--deprecation-folder', '-d', type=Path, required=True,
+              help='Location to move the deprecated rule file to')
 @click.pass_context
-def deprecate_rule(ctx: click.Context, rule_file: Path):
+def deprecate_rule(ctx: click.Context, rule_file: Path, deprecation_folder: Path):
     """Deprecate a rule."""
     version_info = loaded_version_lock.version_lock
     rule_collection = RuleCollection()
     contents = rule_collection.load_file(rule_file).contents
     rule = TOMLRule(path=rule_file, contents=contents)
 
-    if rule.contents.id not in version_info:
+    if rule.contents.id not in version_info and not RULES_CONFIG.bypass_version_lock:
         click.echo('Rule has not been version locked and so does not need to be deprecated. '
-                   'Delete the file or update the maturity to `development` instead')
+                   'Delete the file or update the maturity to `development` instead.')
         ctx.exit()
 
     today = time.strftime('%Y/%m/%d')
-    deprecated_path = rule.get_base_rule_dir() / '_deprecated' / rule_file.name
+    deprecated_path = deprecation_folder / rule_file.name
 
     # create the new rule and save it
     new_meta = dataclasses.replace(rule.contents.metadata,
                                    updated_date=today,
                                    deprecation_date=today,
                                    maturity='deprecated')
     contents = dataclasses.replace(rule.contents, metadata=new_meta)
-    new_rule = TOMLRule(contents=contents, path=Path(deprecated_path))
+    new_rule = TOMLRule(contents=contents, path=deprecated_path)
+    deprecated_path.parent.mkdir(parents=True, exist_ok=True)
     new_rule.save_toml()
 
     # remove the old rule
@@ -817,13 +838,19 @@ def raw_permalink(raw_link):
 @click.argument('stack_version')
 @click.option('--skip-rule-updates', is_flag=True, help='Skip updating the rules')
 @click.option('--dry-run', is_flag=True, help='Print the changes rather than saving the file')
-def trim_version_lock(stack_version: str, skip_rule_updates: bool, dry_run: bool):
+@click.pass_context
+def trim_version_lock(ctx: click.Context, stack_version: str, skip_rule_updates: bool, dry_run: bool):
     """Trim all previous entries within the version lock file which are lower than the min_version."""
     stack_versions = get_stack_versions()
     assert stack_version in stack_versions, \
         f'Unknown min_version ({stack_version}), expected: {", ".join(stack_versions)}'
 
     min_version = Version.parse(stack_version)
+
+    if RULES_CONFIG.bypass_version_lock:
+        click.echo('WARNING: Cannot trim the version lock when the versioning strategy is configured to bypass the '
+                   'version lock. Set `bypass_version_lock` to `false` in the rules config to use the version lock.')
+        ctx.exit()
     version_lock_dict = loaded_version_lock.version_lock.to_dict()
     removed = defaultdict(list)
     rule_msv_drops = []