[8.12] [Security Solution][Endpoint] Fix Manifest Manger so that it works with large (>10k) (#174411)

x-pack/plugins/fleet/server/mocks/index.ts

@@ -31,6 +31,8 @@ import { packageServiceMock } from '../services/epm/package_service.mock';
 import type { UninstallTokenServiceInterface } from '../services/security/uninstall_token_service';
 import type { MessageSigningServiceInterface } from '../services/security';
 
+import { PackagePolicyMocks } from './package_policy.mocks';
+
 // Export all mocks from artifacts
 export * from '../services/artifacts/mocks';
 
@@ -40,6 +42,8 @@ export * from '../services/files/mocks';
 // export all mocks from fleet actions client
 export * from '../services/actions/mocks';
 
+export * from './package_policy.mocks';
+
 export interface MockedFleetAppContext extends FleetAppContext {
   elasticsearch: ReturnType<typeof elasticsearchServiceMock.createStart>;
   data: ReturnType<typeof dataPluginMock.createStartContract>;
@@ -144,6 +148,22 @@ export const createPackagePolicyServiceMock = (): jest.Mocked<PackagePolicyClien
     getUpgradePackagePolicyInfo: jest.fn(),
     enrichPolicyWithDefaultsFromPackage: jest.fn(),
     findAllForAgentPolicy: jest.fn(),
+    fetchAllItems: jest.fn((..._) => {
+      return {
+        async *[Symbol.asyncIterator]() {
+          yield Promise.resolve([PackagePolicyMocks.generatePackagePolicy({ id: '111' })]);
+          yield Promise.resolve([PackagePolicyMocks.generatePackagePolicy({ id: '222' })]);
+        },
+      };
+    }),
+    fetchAllItemIds: jest.fn((..._) => {
+      return {
+        async *[Symbol.asyncIterator]() {
+          yield Promise.resolve(['111']);
+          yield Promise.resolve(['222']);
+        },
+      };
+    }),
   };
 };
 

x-pack/plugins/fleet/server/mocks/package_policy.mocks.ts

@@ -0,0 +1,109 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { SavedObjectsFindResponse } from '@kbn/core-saved-objects-api-server';
+
+import type { SavedObjectsFindResult } from '@kbn/core-saved-objects-api-server';
+
+import { mapPackagePolicySavedObjectToPackagePolicy } from '../services/package_policies';
+
+import type { PackagePolicy } from '../../common';
+import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '../../common';
+
+import type { PackagePolicySOAttributes } from '../types';
+
+const generatePackagePolicySOAttributesMock = (
+  overrides: Partial<PackagePolicySOAttributes> = {}
+): PackagePolicySOAttributes => {
+  return {
+    name: `Package Policy 1`,
+    description: 'Policy for things',
+    created_at: '2024-01-24T15:21:13.389Z',
+    created_by: 'elastic',
+    updated_at: '2024-01-25T15:21:13.389Z',
+    updated_by: 'user-a',
+    policy_id: '444-555-666',
+    enabled: true,
+    inputs: [],
+    namespace: 'default',
+    package: {
+      name: 'endpoint',
+      title: 'Elastic Endpoint',
+      version: '1.0.0',
+    },
+    revision: 1,
+    is_managed: false,
+    secret_references: [],
+    vars: {},
+    elasticsearch: {
+      privileges: {
+        cluster: [],
+      },
+    },
+    agents: 2,
+
+    ...overrides,
+  };
+};
+
+const generatePackagePolicyMock = (overrides: Partial<PackagePolicy> = {}) => {
+  return {
+    ...mapPackagePolicySavedObjectToPackagePolicy(generatePackagePolicySavedObjectMock()),
+    ...overrides,
+  };
+};
+
+const generatePackagePolicySavedObjectMock = (
+  soAttributes: PackagePolicySOAttributes = generatePackagePolicySOAttributesMock()
+): SavedObjectsFindResult<PackagePolicySOAttributes> => {
+  return {
+    score: 1,
+    id: 'so-123',
+    type: PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+    version: 'abc',
+    created_at: soAttributes.created_at,
+    updated_at: soAttributes.updated_at,
+    attributes: soAttributes,
+    references: [],
+    sort: ['created_at'],
+  };
+};
+
+const generatePackagePolicySavedObjectFindResponseMock = (
+  soResults?: PackagePolicySOAttributes[]
+): SavedObjectsFindResponse<PackagePolicySOAttributes> => {
+  const soList = soResults ?? [
+    generatePackagePolicySOAttributesMock(),
+    generatePackagePolicySOAttributesMock(),
+  ];
+
+  return {
+    saved_objects: soList.map((soAttributes) => {
+      return {
+        score: 1,
+        id: 'so-123',
+        type: PACKAGE_POLICY_SAVED_OBJECT_TYPE,
+        version: 'abc',
+        created_at: soAttributes.created_at,
+        updated_at: soAttributes.updated_at,
+        attributes: soAttributes,
+        references: [],
+        sort: ['created_at'],
+      };
+    }),
+    total: soList.length,
+    per_page: 10,
+    page: 1,
+    pit_id: 'pit-id-1',
+  };
+};
+
+export const PackagePolicyMocks = Object.freeze({
+  generatePackagePolicySOAttributes: generatePackagePolicySOAttributesMock,
+  generatePackagePolicySavedObjectFindResponse: generatePackagePolicySavedObjectFindResponseMock,
+  generatePackagePolicy: generatePackagePolicyMock,
+});

x-pack/plugins/fleet/server/services/artifacts/artifacts.test.ts

@@ -11,6 +11,8 @@ import { errors } from '@elastic/elasticsearch';
 
 import type { TransportResult } from '@elastic/elasticsearch';
 
+import { set } from '@kbn/safer-lodash-set';
+
 import { FLEET_SERVER_ARTIFACTS_INDEX } from '../../../common';
 
 import { ArtifactsElasticsearchError } from '../../errors';
@@ -33,12 +35,14 @@ import {
   createArtifact,
   deleteArtifact,
   encodeArtifactContent,
+  fetchAllArtifacts,
   generateArtifactContentHash,
   getArtifact,
   listArtifacts,
 } from './artifacts';
 
 import type { NewArtifact } from './types';
+import type { FetchAllArtifactsOptions } from './types';
 
 describe('When using the artifacts services', () => {
   let esClientMock: ReturnType<typeof elasticsearchServiceMock.createInternalClient>;
@@ -324,8 +328,28 @@ describe('When using the artifacts services', () => {
         newArtifact,
       ]);
 
-      expect(responseErrors).toEqual([new Error('error')]);
-      expect(artifacts).toBeUndefined();
+      expect(responseErrors).toEqual([
+        new Error(
+          'Create of artifact id [undefined] returned: result [undefined], status [400], reason [{"reason":"error"}]'
+        ),
+      ]);
+      expect(artifacts).toEqual([
+        {
+          body: 'eJyrVkrNKynKTC1WsoqOrQUAJxkFKQ==',
+          compressionAlgorithm: 'zlib',
+          created: expect.any(String),
+          decodedSha256: 'd801aa1fb',
+          decodedSize: 14,
+          encodedSha256: 'd29238d40',
+          encodedSize: 22,
+          encryptionAlgorithm: 'none',
+          id: 'endpoint:trustlist-v1-d801aa1fb',
+          identifier: 'trustlist-v1',
+          packageName: 'endpoint',
+          relative_url: '/api/fleet/artifacts/trustlist-v1/d801aa1fb',
+          type: 'trustlist',
+        },
+      ]);
     });
   });
 
@@ -488,4 +512,113 @@ describe('When using the artifacts services', () => {
       });
     });
   });
+
+  describe('and calling `fetchAll()`', () => {
+    beforeEach(() => {
+      esClientMock.search
+        .mockResolvedValueOnce(generateArtifactEsSearchResultHitsMock())
+        .mockResolvedValueOnce(generateArtifactEsSearchResultHitsMock())
+        .mockResolvedValueOnce(set(generateArtifactEsSearchResultHitsMock(), 'hits.hits', []));
+    });
+
+    it('should return an iterator', async () => {
+      expect(fetchAllArtifacts(esClientMock)).toEqual({
+        [Symbol.asyncIterator]: expect.any(Function),
+      });
+    });
+
+    it('should provide artifacts on each iteration', async () => {
+      for await (const artifacts of fetchAllArtifacts(esClientMock)) {
+        expect(artifacts[0]).toEqual({
+          body: expect.anything(),
+          compressionAlgorithm: expect.anything(),
+          created: expect.anything(),
+          decodedSha256: expect.anything(),
+          decodedSize: expect.anything(),
+          encodedSha256: expect.anything(),
+          encodedSize: expect.anything(),
+          encryptionAlgorithm: expect.anything(),
+          id: expect.anything(),
+          identifier: expect.anything(),
+          packageName: expect.anything(),
+          relative_url: expect.anything(),
+          type: expect.anything(),
+        });
+      }
+
+      expect(esClientMock.search).toHaveBeenCalledTimes(3);
+    });
+
+    it('should use defaults if no `options` were provided', async () => {
+      for await (const artifacts of fetchAllArtifacts(esClientMock)) {
+        expect(artifacts.length).toBeGreaterThan(0);
+      }
+
+      expect(esClientMock.search).toHaveBeenLastCalledWith(
+        expect.objectContaining({
+          q: '',
+          size: 1000,
+          sort: [{ created: { order: 'asc' } }],
+          _source_excludes: undefined,
+        })
+      );
+    });
+
+    it('should use custom options when provided', async () => {
+      const options: FetchAllArtifactsOptions = {
+        kuery: 'foo: something',
+        sortOrder: 'desc',
+        perPage: 500,
+        sortField: 'someField',
+        includeArtifactBody: false,
+      };
+
+      for await (const artifacts of fetchAllArtifacts(esClientMock, options)) {
+        expect(artifacts.length).toBeGreaterThan(0);
+      }
+
+      expect(esClientMock.search).toHaveBeenCalledWith(
+        expect.objectContaining({
+          q: options.kuery,
+          size: options.perPage,
+          sort: [{ [options.sortField!]: { order: options.sortOrder } }],
+          _source_excludes: 'body',
+        })
+      );
+    });
+
+    it('should set `done` to true if loop `break`s out', async () => {
+      const iterator = fetchAllArtifacts(esClientMock);
+
+      for await (const _ of iterator) {
+        break;
+      }
+
+      await expect(iterator[Symbol.asyncIterator]().next()).resolves.toEqual({
+        done: true,
+        value: expect.any(Array),
+      });
+
+      expect(esClientMock.search).toHaveBeenCalledTimes(1);
+    });
+
+    it('should handle throwing in loop by setting `done` to `true`', async () => {
+      const iterator = fetchAllArtifacts(esClientMock);
+
+      try {
+        for await (const _ of iterator) {
+          throw new Error('test');
+        }
+      } catch (e) {
+        expect(e); // just to silence eslint
+      }
+
+      await expect(iterator[Symbol.asyncIterator]().next()).resolves.toEqual({
+        done: true,
+        value: expect.any(Array),
+      });
+
+      expect(esClientMock.search).toHaveBeenCalledTimes(1);
+    });
+  });
 });