feat: use up to 110K keys when purging storage

2anki · Apr 1, 2023 · 3dd38b4 · 3dd38b4
1 parent 9a17b31
commit 3dd38b4
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 3 deletions.
diff --git a/src/lib/storage/StorageHandler.ts b/src/lib/storage/StorageHandler.ts
@@ -57,11 +57,11 @@ class StorageHandler {
     });
   }
 
-  getContents(): Promise<ObjectList | undefined> {
+  getContents(maxKeys: number = 1000): Promise<ObjectList | undefined> {
     const { s3 } = this;
     return new Promise((resolve, reject) => {
       s3.listObjects(
-        { Bucket: StorageHandler.DefaultBucketName() },
+        { Bucket: StorageHandler.DefaultBucketName(), MaxKeys: maxKeys },
         (err, data) => {
           if (err) {
             sendError(err);

diff --git a/src/lib/storage/jobs/helpers/deleteOldUploads.ts b/src/lib/storage/jobs/helpers/deleteOldUploads.ts
@@ -7,6 +7,7 @@ import StorageHandler from '../../StorageHandler';
 import { Upload } from '../../types';
 
 export const MS_21 = TIME_21_MINUTES_AS_SECONDS * 1000;
+const MAX_KEYS = 110_000;
 
 const getFreeUsers = (db: Knex): Promise<Users[]> =>
   db('users').where('patreon', 'false').returning(['owner']);
@@ -27,7 +28,7 @@ export default async function deleteOldUploads(db: Knex) {
     await db('uploads').delete().where('key', upload.key);
   }
 
-  const files = await storage.getContents();
+  const files = await storage.getContents(MAX_KEYS);
   const query = await db.raw(`
     SELECT up.key FROM users u JOIN uploads up ON u.id = up.owner WHERE u.patreon = true;
   `);