feat: increase targets limit and add pagingation (#86)

* feat: increase targets limit and add pagingation * feat: small clean up
snyk-tech-services · Dec 4, 2023 · c91e91a · c91e91a
1 parent c3d8364
commit c91e91a
Show file tree

Hide file tree

Showing 3 changed files with 95 additions and 20 deletions.
diff --git a/src/lib/snyk/index.ts b/src/lib/snyk/index.ts
@@ -1,4 +1,4 @@
-import { Orgs, Org } from 'snyk-api-ts-client';
+import { Org } from 'snyk-api-ts-client';
 import { requestsManager } from 'snyk-request-manager';
 import * as debugLib from 'debug';
 import { Integration } from '../types';
@@ -112,27 +112,40 @@ export const retrieveMonitoredReposBySourceType = async (
     const snykRequestManager = new requestsManager();
 
     for (let i = 0; i < orgs.length; i++) {
-      const requestSync = await snykRequestManager.request({
-        verb: 'GET',
-        url: `/orgs/${orgs[i].id}/targets?origin=${SourceType[sourceType]}&version=${snykApiVersion}`,
-        useRESTApi: true,
-      });
+      let isNextPage = true;
+      let url = `/orgs/${orgs[i].id}/targets?limit=100&origin=${SourceType[sourceType]}&version=${snykApiVersion}`;
+      while (isNextPage) {
+        const targetsResponse = await snykRequestManager.request({
+          url,
+          verb: 'GET',
+          useRESTApi: true,
+        });
 
-      let targets = requestSync.data.data as TargetType[];
+        let targets = targetsResponse.data.data as TargetType[];
 
-      if (SourceType[sourceType] === 'cli' && scmHostname) {
-        targets = targets.filter(
-          (target: TargetType) =>
-            target.attributes.remoteUrl &&
-            target.attributes.remoteUrl.includes(scmHostname),
+        if (SourceType[sourceType] === 'cli' && scmHostname) {
+          targets = targets.filter(
+            (target: TargetType) =>
+              target.attributes.remoteUrl &&
+              target.attributes.remoteUrl.includes(scmHostname),
+          );
+        }
+        const targetDisplayNames = targets.map(
+          (target) => target.attributes.displayName,
         );
-      }
-      const targetDisplayNames = targets.map(
-        (target) => target.attributes.displayName,
-      );
 
-      snykScmMonitoredRepos = snykScmMonitoredRepos.concat(targetDisplayNames);
+        snykScmMonitoredRepos =
+          snykScmMonitoredRepos.concat(targetDisplayNames);
+
+        if (targetsResponse.data?.links?.next) {
+          console.log('fetching next page');
+          url = targetsResponse.data.links.next;
+        } else {
+          isNextPage = false;
+        }
+      }
     }
+
     return snykScmMonitoredRepos;
   } catch (err: any) {
     debug('Failed retrieving Snyk monitored SCM repos\n' + err);

diff --git a/test/fixtures/snyk/targetMock.ts b/test/fixtures/snyk/targetMock.ts
@@ -1,4 +1,4 @@
-import { TargetType } from '../../../src/lib/snyk';
+import { snykApiVersion, TargetType } from '../../../src/lib/snyk';
 
 export const listAllTargetCliOnly: { data: TargetType[] } = {
   data: [
@@ -59,3 +59,36 @@ export const listAllTargetsScmOnly = {
     },
   ],
 };
+
+export const listTargetsWithNextPage = {
+  data: [
+    {
+      type: 'target',
+      id: 'ef5397fc-a8c0-4ee3-b7c8-e05902a982d6',
+      attributes: {
+        isPrivate: true,
+        origin: 'bitbucket-server',
+        displayName: 'test-snyk-first-page',
+        remoteUrl: null,
+      },
+    },
+  ],
+  links: {
+    next: `/orgs/39ab9ba8-96e4-41b5-8494-4fe31bf8907a/targets?limit=100&origin=bitbucket-server&version=${snykApiVersion}&starting_after=doesnt-matter`,
+  },
+};
+
+export const listTargetsLastPage = {
+  data: [
+    {
+      type: 'target',
+      id: 'ef5397fc-a8c0-4ee3-b7c8-e05902a982d6',
+      attributes: {
+        isPrivate: true,
+        origin: 'bitbucket-server',
+        displayName: 'test-snyk-last-page',
+        remoteUrl: null,
+      },
+    },
+  ],
+};
diff --git a/test/lib/snyk/index.test.ts b/test/lib/snyk/index.test.ts
@@ -3,6 +3,8 @@ import * as snyk from '../../../src/lib/snyk';
 import {
   listAllTargetCliOnly,
   listAllTargetsScmOnly,
+  listTargetsLastPage,
+  listTargetsWithNextPage,
 } from '../../fixtures/snyk/targetMock';
 import { snykApiVersion } from '../../../src/lib/snyk';
 
@@ -12,10 +14,14 @@ beforeEach(() => {
     .get(/.*/)
     .reply(200, (uri) => {
       switch (uri) {
-        case `/rest/orgs/689ce7f9-7943-4a71-b704-2ba575f01088/targets?origin=cli&version=${snykApiVersion}`:
+        case `/rest/orgs/689ce7f9-7943-4a71-b704-2ba575f01088/targets?limit=100&origin=cli&version=${snykApiVersion}`:
           return listAllTargetCliOnly;
-        case `/rest/orgs/689ce7f9-7943-4a71-b704-2ba575f01089/targets?origin=bitbucket-server&version=${snykApiVersion}`:
+        case `/rest/orgs/689ce7f9-7943-4a71-b704-2ba575f01089/targets?limit=100&origin=bitbucket-server&version=${snykApiVersion}`:
           return listAllTargetsScmOnly;
+        case `/rest/orgs/39ab9ba8-96e4-41b5-8494-4fe31bf8907a/targets?limit=100&origin=bitbucket-server&version=${snykApiVersion}`:
+          return listTargetsWithNextPage;
+        case `/rest/orgs/39ab9ba8-96e4-41b5-8494-4fe31bf8907a/targets?limit=100&origin=bitbucket-server&version=${snykApiVersion}&starting_after=doesnt-matter`:
+          return listTargetsLastPage;
         default:
       }
     });
@@ -89,6 +95,29 @@ describe('Testing snyk lib functions', () => {
     ).toEqual(['test-snyk', 'test-snyk-2']);
   });
 
+  test('Retrieve Snyk monitored repos with pagination - bitbucket-server project', async () => {
+    const orgs: snyk.OrgType[] = [
+      {
+        id: '39ab9ba8-96e4-41b5-8494-4fe31bf8907a',
+        type: 'org',
+        attributes: {
+          name: 'defaultOrg',
+          slug: 'default-org',
+          group: null,
+          is_personal: false,
+        },
+      },
+    ];
+
+    const response = await snyk.retrieveMonitoredReposBySourceType(
+      orgs,
+      snyk.SourceType['bitbucket-server'],
+      'http://123',
+    );
+
+    expect(response).toEqual(['test-snyk-first-page', 'test-snyk-last-page']);
+  });
+
   // TODO: See how to mock Snyk API better to honor filters passed in requestBody
 
   //   test('Retrieve Snyk monitored repos - CLI projects mismatching SCM hostname', async () => {