[Security Solution][Exceptions] Use semantic version for manifest ver…

…sion + Scaling Tweaks (#73388)

* Manifest version is semantic version

* Configurable task interval

* Use task interval over scheduled when provided

* Fix crash on download of large artifact

* Don't need to generate linux artifacts

* Configurable artifact validation

* Test fixes

* Test fixes

* Type/test fixes

* Final tweaks

* Remove linux endpoint exception generation from UI

* Fix paging so that we stop before 10k

* Fix pagination

* Fix pagination test

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -1041,7 +1041,7 @@ export class EndpointDocGenerator {
           config: {
             artifact_manifest: {
               value: {
-                manifest_version: 'WzAsMF0=',
+                manifest_version: '1.0.0',
                 schema_version: 'v1',
                 artifacts: {},
               },

x-pack/plugins/security_solution/common/endpoint/schema/common.ts

@@ -23,8 +23,6 @@ export const encryptionAlgorithm = t.keyof({
 
 export const identifier = t.string;
 
-export const manifestVersion = t.string;
-
 export const manifestSchemaVersion = t.keyof({
   v1: null,
 });
@@ -34,4 +32,7 @@ export const relativeUrl = t.string;
 
 export const sha256 = t.string;
 
+export const semanticVersion = t.string;
+export type SemanticVersion = t.TypeOf<typeof semanticVersion>;
+
 export const size = t.number;

x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts

@@ -11,9 +11,9 @@ import {
   encryptionAlgorithm,
   identifier,
   manifestSchemaVersion,
-  manifestVersion,
   relativeUrl,
   sha256,
+  semanticVersion,
   size,
 } from './common';
 
@@ -50,7 +50,7 @@ export type ManifestEntryDispatchSchema = t.TypeOf<typeof manifestEntryDispatchS
 
 export const manifestBaseSchema = t.exact(
   t.type({
-    manifest_version: manifestVersion,
+    manifest_version: semanticVersion,
     schema_version: manifestSchemaVersion,
   })
 );

x-pack/plugins/security_solution/public/common/components/exceptions/add_exception_modal/index.tsx

@@ -234,7 +234,7 @@ export const AddExceptionModal = memo(function AddExceptionModal({
   );
 
   const retrieveAlertOsTypes = useCallback(() => {
-    const osDefaults = ['windows', 'macos', 'linux'];
+    const osDefaults = ['windows', 'macos'];
     if (alertData) {
       const osTypes = getMappedNonEcsValue({
         data: alertData.nonEcsData,

x-pack/plugins/security_solution/server/config.ts

@@ -29,6 +29,12 @@ export const configSchema = schema.object({
     from: schema.string({ defaultValue: 'now-15m' }),
     to: schema.string({ defaultValue: 'now' }),
   }),
+
+  /**
+   * Artifacts Configuration
+   */
+  packagerTaskInterval: schema.string({ defaultValue: '60s' }),
+  validateArtifactDownloads: schema.boolean({ defaultValue: true }),
 });
 
 export const createConfig$ = (context: PluginInitializerContext) =>

x-pack/plugins/security_solution/server/endpoint/config.ts

@@ -27,6 +27,12 @@ export const EndpointConfigSchema = schema.object({
     from: schema.string({ defaultValue: 'now-15m' }),
     to: schema.string({ defaultValue: 'now' }),
   }),
+
+  /**
+   * Artifacts Configuration
+   */
+  packagerTaskInterval: schema.string({ defaultValue: '60s' }),
+  validateArtifactDownloads: schema.boolean({ defaultValue: true }),
 });
 
 export function createConfig$(context: PluginInitializerContext) {

x-pack/plugins/security_solution/server/endpoint/ingest_integration.test.ts

@@ -12,7 +12,6 @@ import {
   ManifestManagerMockType,
 } from './services/artifacts/manifest_manager/manifest_manager.mock';
 import { getPackageConfigCreateCallback } from './ingest_integration';
-import { ManifestConstants } from './lib/artifacts';
 
 describe('ingest_integration tests ', () => {
   describe('ingest_integration sanity checks', () => {
@@ -30,16 +29,6 @@ describe('ingest_integration tests ', () => {
       expect(newPolicyConfig.inputs[0]!.config!.policy.value).toEqual(policyConfigFactory());
       expect(newPolicyConfig.inputs[0]!.config!.artifact_manifest.value).toEqual({
         artifacts: {
-          'endpoint-exceptionlist-linux-v1': {
-            compression_algorithm: 'zlib',
-            decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
-            decoded_size: 14,
-            encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
-            encoded_size: 22,
-            encryption_algorithm: 'none',
-            relative_url:
-              '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
-          },
           'endpoint-exceptionlist-macos-v1': {
             compression_algorithm: 'zlib',
             decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
@@ -61,7 +50,7 @@ describe('ingest_integration tests ', () => {
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
           },
         },
-        manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537',
+        manifest_version: '1.0.0',
         schema_version: 'v1',
       });
     });
@@ -70,9 +59,7 @@ describe('ingest_integration tests ', () => {
       const logger = loggingSystemMock.create().get('ingest_integration.test');
       const manifestManager = getManifestManagerMock();
       manifestManager.pushArtifacts = jest.fn().mockResolvedValue([new Error('error updating')]);
-      const lastComputed = await manifestManager.getLastComputedManifest(
-        ManifestConstants.SCHEMA_VERSION
-      );
+      const lastComputed = await manifestManager.getLastComputedManifest();
 
       const callback = getPackageConfigCreateCallback(logger, manifestManager);
       const policyConfig = createNewPackageConfigMock();
@@ -90,9 +77,7 @@ describe('ingest_integration tests ', () => {
       const manifestManager = getManifestManagerMock({
         mockType: ManifestManagerMockType.InitialSystemState,
       });
-      const lastComputed = await manifestManager.getLastComputedManifest(
-        ManifestConstants.SCHEMA_VERSION
-      );
+      const lastComputed = await manifestManager.getLastComputedManifest();
       expect(lastComputed).toEqual(null);
 
       manifestManager.buildNewManifest = jest.fn().mockRejectedValue(new Error('abcd'));
@@ -107,9 +92,7 @@ describe('ingest_integration tests ', () => {
     test('subsequent policy creations succeed', async () => {
       const logger = loggingSystemMock.create().get('ingest_integration.test');
       const manifestManager = getManifestManagerMock();
-      const lastComputed = await manifestManager.getLastComputedManifest(
-        ManifestConstants.SCHEMA_VERSION
-      );
+      const lastComputed = await manifestManager.getLastComputedManifest();
 
       manifestManager.buildNewManifest = jest.fn().mockResolvedValue(lastComputed); // no diffs
       const callback = getPackageConfigCreateCallback(logger, manifestManager);

x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts

@@ -10,22 +10,22 @@ import { factory as policyConfigFactory } from '../../common/endpoint/models/pol
 import { NewPolicyData } from '../../common/endpoint/types';
 import { ManifestManager } from './services/artifacts';
 import { Manifest } from './lib/artifacts';
-import { reportErrors, ManifestConstants } from './lib/artifacts/common';
+import { reportErrors } from './lib/artifacts/common';
 import { InternalArtifactCompleteSchema } from './schemas/artifacts';
 import { manifestDispatchSchema } from '../../common/endpoint/schema/manifest';
 
 const getManifest = async (logger: Logger, manifestManager: ManifestManager): Promise<Manifest> => {
   let manifest: Manifest | null = null;
 
   try {
-    manifest = await manifestManager.getLastComputedManifest(ManifestConstants.SCHEMA_VERSION);
+    manifest = await manifestManager.getLastComputedManifest();
 
     // If we have not yet computed a manifest, then we have to do so now. This should only happen
     // once.
     if (manifest == null) {
       // New computed manifest based on current state of exception list
-      const newManifest = await manifestManager.buildNewManifest(ManifestConstants.SCHEMA_VERSION);
-      const diffs = newManifest.diff(Manifest.getDefault(ManifestConstants.SCHEMA_VERSION));
+      const newManifest = await manifestManager.buildNewManifest();
+      const diffs = newManifest.diff(Manifest.getDefault());
 
       // Compress new artifacts
       const adds = diffs.filter((diff) => diff.type === 'add').map((diff) => diff.id);
@@ -63,7 +63,7 @@ const getManifest = async (logger: Logger, manifestManager: ManifestManager): Pr
     logger.error(err);
   }
 
-  return manifest ?? Manifest.getDefault(ManifestConstants.SCHEMA_VERSION);
+  return manifest ?? Manifest.getDefault();
 };
 
 /**

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts

@@ -13,13 +13,11 @@ import {
 export const ArtifactConstants = {
   GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist',
   SAVED_OBJECT_TYPE: 'endpoint:user-artifact',
-  SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'],
-  SCHEMA_VERSION: 'v1',
+  SUPPORTED_OPERATING_SYSTEMS: ['macos', 'windows'],
 };
 
 export const ManifestConstants = {
   SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest',
-  SCHEMA_VERSION: 'v1',
 };
 
 export const getArtifactId = (artifact: InternalArtifactSchema) => {

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts

@@ -314,21 +314,23 @@ describe('buildEventTypeSignal', () => {
   test('it should convert the exception lists response to the proper endpoint format while paging', async () => {
     // The first call returns two exceptions
     const first = getFoundExceptionListItemSchemaMock();
+    first.per_page = 2;
+    first.total = 4;
     first.data.push(getExceptionListItemSchemaMock());
 
     // The second call returns two exceptions
     const second = getFoundExceptionListItemSchemaMock();
+    second.per_page = 2;
+    second.total = 4;
     second.data.push(getExceptionListItemSchemaMock());
 
-    // The third call returns no exceptions, paging stops
-    const third = getFoundExceptionListItemSchemaMock();
-    third.data = [];
     mockExceptionClient.findExceptionListItem = jest
       .fn()
       .mockReturnValueOnce(first)
-      .mockReturnValueOnce(second)
-      .mockReturnValueOnce(third);
+      .mockReturnValueOnce(second);
+
     const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
+
     // Expect 2 exceptions, the first two calls returned the same exception list items
     expect(resp.entries.length).toEqual(2);
   });

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts

@@ -79,10 +79,10 @@ export async function getFullEndpointExceptionList(
   schemaVersion: string
 ): Promise<WrappedTranslatedExceptionList> {
   const exceptions: WrappedTranslatedExceptionList = { entries: [] };
-  let numResponses = 0;
   let page = 1;
+  let paging = true;
 
-  do {
+  while (paging) {
     const response = await eClient.findExceptionListItem({
       listId: ENDPOINT_LIST_ID,
       namespaceType: 'agnostic',
@@ -94,17 +94,16 @@ export async function getFullEndpointExceptionList(
     });
 
     if (response?.data !== undefined) {
-      numResponses = response.data.length;
-
       exceptions.entries = exceptions.entries.concat(
         translateToEndpointExceptions(response, schemaVersion)
       );
 
+      paging = (page - 1) * 100 + response.data.length < response.total;
       page++;
     } else {
       break;
     }
-  } while (numResponses > 0);
+  }
 
   const [validated, errors] = validate(exceptions, wrappedTranslatedExceptionList);
   if (errors != null) {

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts

@@ -6,7 +6,7 @@
 
 import { ManifestSchemaVersion } from '../../../../common/endpoint/schema/common';
 import { InternalArtifactCompleteSchema } from '../../schemas';
-import { ManifestConstants, getArtifactId } from './common';
+import { getArtifactId } from './common';
 import { Manifest } from './manifest';
 import {
   getMockArtifacts,
@@ -30,29 +30,21 @@ describe('manifest', () => {
     });
 
     test('Can create manifest with valid schema version', () => {
-      const manifest = new Manifest('v1');
+      const manifest = new Manifest();
       expect(manifest).toBeInstanceOf(Manifest);
     });
 
     test('Cannot create manifest with invalid schema version', () => {
       expect(() => {
-        new Manifest('abcd' as ManifestSchemaVersion);
+        new Manifest({
+          schemaVersion: 'abcd' as ManifestSchemaVersion,
+        });
       }).toThrow();
     });
 
     test('Empty manifest transforms correctly to expected endpoint format', async () => {
       expect(emptyManifest.toEndpointFormat()).toStrictEqual({
         artifacts: {
-          'endpoint-exceptionlist-linux-v1': {
-            compression_algorithm: 'zlib',
-            encryption_algorithm: 'none',
-            decoded_sha256: 'd801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
-            encoded_sha256: 'f8e6afa1d5662f5b37f83337af774b5785b5b7f1daee08b7b00c2d6813874cda',
-            decoded_size: 14,
-            encoded_size: 22,
-            relative_url:
-              '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
-          },
           'endpoint-exceptionlist-macos-v1': {
             compression_algorithm: 'zlib',
             encryption_algorithm: 'none',
@@ -74,24 +66,14 @@ describe('manifest', () => {
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658',
           },
         },
-        manifest_version: 'a9b7ef358a363f327f479e31efc4f228b2277a7fb4d1914ca9b4e7ca9ffcf537',
+        manifest_version: '1.0.0',
         schema_version: 'v1',
       });
     });
 
     test('Manifest transforms correctly to expected endpoint format', async () => {
       expect(manifest1.toEndpointFormat()).toStrictEqual({
         artifacts: {
-          'endpoint-exceptionlist-linux-v1': {
-            compression_algorithm: 'zlib',
-            encryption_algorithm: 'none',
-            decoded_sha256: '96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
-            encoded_sha256: '975382ab55d019cbab0bbac207a54e2a7d489fad6e8f6de34fc6402e5ef37b1e',
-            decoded_size: 432,
-            encoded_size: 147,
-            relative_url:
-              '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
-          },
           'endpoint-exceptionlist-macos-v1': {
             compression_algorithm: 'zlib',
             encryption_algorithm: 'none',
@@ -113,15 +95,16 @@ describe('manifest', () => {
               '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
           },
         },
-        manifest_version: 'a7f4760bfa2662e85e30fe4fb8c01b4c4a20938c76ab21d3c5a3e781e547cce7',
+        manifest_version: '1.0.0',
         schema_version: 'v1',
       });
     });
 
     test('Manifest transforms correctly to expected saved object format', async () => {
       expect(manifest1.toSavedObject()).toStrictEqual({
+        schemaVersion: 'v1',
+        semanticVersion: '1.0.0',
         ids: [
-          'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
           'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
           'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
         ],
@@ -133,12 +116,12 @@ describe('manifest', () => {
       expect(diffs).toEqual([
         {
           id:
-            'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
+            'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
           type: 'delete',
         },
         {
           id:
-            'endpoint-exceptionlist-linux-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8',
+            'endpoint-exceptionlist-macos-v1-0a5a2013a79f9e60682472284a1be45ab1ff68b9b43426d00d665016612c15c8',
           type: 'add',
         },
       ]);
@@ -154,7 +137,6 @@ describe('manifest', () => {
       const entries = manifest1.getEntries();
       const keys = Object.keys(entries);
       expect(keys).toEqual([
-        'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
         'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
         'endpoint-exceptionlist-windows-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3',
       ]);
@@ -168,13 +150,8 @@ describe('manifest', () => {
     });
 
     test('Manifest can be created from list of artifacts', async () => {
-      const oldManifest = new Manifest(ManifestConstants.SCHEMA_VERSION);
-      const manifest = Manifest.fromArtifacts(artifacts, 'v1', oldManifest);
-      expect(
-        manifest.contains(
-          'endpoint-exceptionlist-linux-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'
-        )
-      ).toEqual(true);
+      const oldManifest = new Manifest();
+      const manifest = Manifest.fromArtifacts(artifacts, oldManifest);
       expect(
         manifest.contains(
           'endpoint-exceptionlist-macos-v1-96b76a1a911662053a1562ac14c4ff1e87c2ff550d6fe52e1e0b3790526597d3'