Store cache control headers, change index order

synzen · Dec 31, 2024 · 0dd2fd4 · 0dd2fd4
1 parent 4fbadde
commit 0dd2fd4
Show file tree

Hide file tree

Showing 4 changed files with 42 additions and 32 deletions.
diff --git a/services/feed-requests/migrations/Migration20241231120419.ts b/services/feed-requests/migrations/Migration20241231120419.ts
@@ -0,0 +1,19 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20241231120419 extends Migration {
+
+  async up(): Promise<void> {
+    this.addSql(`ALTER TYPE request_partitioned_status ADD VALUE 'CACHE_CONTROL_SKIPPED';`);
+    // switch the created_at and status of the index
+    this.addSql(`DROP INDEX request_partitioned_lookupkey_created_at_status_index;`);
+    this.addSql(`CREATE INDEX request_partitioned_lookupkey_status_created_at_index ON request_partitioned (lookup_key, status, created_at);`);
+  }
+
+  async down(): Promise<void> {
+    this.addSql(`ALTER TYPE request_partitioned_status DROP VALUE 'CACHE_CONTROL_SKIPPED';`);
+    // switch the created_at and status of the index
+    this.addSql(`DROP INDEX request_partitioned_lookupkey_status_created_at_index;`);
+    this.addSql(`CREATE INDEX request_partitioned_lookupkey_created_at_status_index ON request_partitioned (lookup_key, created_at, status);`);
+  }
+
+}
diff --git a/services/feed-requests/src/feed-fetcher/entities/response.entity.ts b/services/feed-requests/src/feed-fetcher/entities/response.entity.ts
@@ -49,10 +49,7 @@ export class Response {
     type: 'json',
     nullable: true,
   })
-  headers?: {
-    etag?: string;
-    lastModified?: string;
-  } | null;
+  headers?: Record<string, string> | null;
 
   @Property({
     type: 'timestamp with time zone',

diff --git a/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts b/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts
@@ -58,7 +58,7 @@ interface FetchOptions {
 interface FetchResponse {
   ok: boolean;
   status: number;
-  headers: Map<'etag' | 'last-modified' | 'server' | 'content-type', string>;
+  headers: Map<string, string>;
   text: () => Promise<string>;
   arrayBuffer: () => Promise<ArrayBuffer>;
 }
@@ -135,19 +135,11 @@ export class FeedFetcherService {
     request: Request;
     decodedResponseText: string | null | undefined;
   } | null> {
-    const logDebug =
-      url ===
-      'https://www.clanaod.net/forums/external.php?type=RSS2&forumids=102';
-
     const request = await this.partitionedRequestsStore.getLatestRequest(
       lookupKey || url,
     );
 
     if (!request) {
-      if (logDebug) {
-        logger.warn(`Running debug on schedule: no request was found`);
-      }
-
       return null;
     }
 
@@ -162,13 +154,6 @@ export class FeedFetcherService {
           ).toString()
         : '';
 
-      if (logDebug) {
-        logger.warn(
-          `Running debug on schedule: got cache key ${request.response.redisCacheKey}`,
-          { text },
-        );
-      }
-
       return {
         request,
         decodedResponseText: text,
@@ -232,20 +217,13 @@ export class FeedFetcherService {
         request.status = RequestStatus.BAD_STATUS_CODE;
       }
 
-      const etag = res.headers.get('etag');
-      const lastModified = res.headers.get('last-modified');
-
       const response = new Response();
       response.createdAt = request.createdAt;
       response.statusCode = res.status;
       response.headers = {};
 
-      if (etag) {
-        response.headers.etag = etag;
-      }
-
-      if (lastModified) {
-        response.headers.lastModified = lastModified;
+      for (const [key, val] of res.headers.entries()) {
+        response.headers[key] = val;
       }
 
       let text: string | null = null;
@@ -459,6 +437,11 @@ export class FeedFetcherService {
       convertHeaderValue(normalizedHeaders.get('last-modified')),
     );
     headers.set('server', convertHeaderValue(normalizedHeaders.get('server')));
+    headers.set('date', convertHeaderValue(normalizedHeaders.get('date')));
+    headers.set(
+      'cache-control',
+      convertHeaderValue(normalizedHeaders.get('cache-control')),
+    );
 
     return {
       headers,

diff --git a/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts b/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts
@@ -120,11 +120,19 @@ export default class PartitionedRequestsStoreService {
 
   async getLatestOkRequest(
     lookupKey: string,
-  ): Promise<null | { createdAt: Date }> {
+    opts?: {
+      fields?: Record<'response_headers', boolean>;
+    },
+  ): Promise<null | {
+    createdAt: Date;
+    responseHeaders?: Record<string, string>;
+  }> {
     const em = this.orm.em.getConnection();
 
     const [result] = await em.execute(
-      `SELECT created_at FROM request_partitioned
+      `SELECT created_at ${
+        opts?.fields?.response_headers ? ', response_headers' : ''
+      } FROM request_partitioned
        WHERE lookup_key = ?
        AND status = 'OK'
        ORDER BY created_at DESC
@@ -136,7 +144,10 @@ export default class PartitionedRequestsStoreService {
       return null;
     }
 
-    return { createdAt: new Date(result.created_at) };
+    return {
+      createdAt: new Date(result.created_at),
+      responseHeaders: result.response_headers,
+    };
   }
 
   async countFailedRequests(lookupKey: string, since?: Date) {