Revert "Store cache control headers, change index order"

This reverts commit 0dd2fd4. Original index order was required
synzen · Dec 31, 2024 · 109f0fb · 109f0fb
1 parent 0dd2fd4
commit 109f0fb
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 42 deletions.
diff --git a/services/feed-requests/migrations/Migration20241231120419.ts b/services/feed-requests/migrations/Migration20241231120419.ts
diff --git a/services/feed-requests/src/feed-fetcher/entities/response.entity.ts b/services/feed-requests/src/feed-fetcher/entities/response.entity.ts
@@ -49,7 +49,10 @@ export class Response {
     type: 'json',
     nullable: true,
   })
-  headers?: Record<string, string> | null;
+  headers?: {
+    etag?: string;
+    lastModified?: string;
+  } | null;
 
   @Property({
     type: 'timestamp with time zone',

diff --git a/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts b/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts
@@ -58,7 +58,7 @@ interface FetchOptions {
 interface FetchResponse {
   ok: boolean;
   status: number;
-  headers: Map<string, string>;
+  headers: Map<'etag' | 'last-modified' | 'server' | 'content-type', string>;
   text: () => Promise<string>;
   arrayBuffer: () => Promise<ArrayBuffer>;
 }
@@ -135,11 +135,19 @@ export class FeedFetcherService {
     request: Request;
     decodedResponseText: string | null | undefined;
   } | null> {
+    const logDebug =
+      url ===
+      'https://www.clanaod.net/forums/external.php?type=RSS2&forumids=102';
+
     const request = await this.partitionedRequestsStore.getLatestRequest(
       lookupKey || url,
     );
 
     if (!request) {
+      if (logDebug) {
+        logger.warn(`Running debug on schedule: no request was found`);
+      }
+
       return null;
     }
 
@@ -154,6 +162,13 @@ export class FeedFetcherService {
           ).toString()
         : '';
 
+      if (logDebug) {
+        logger.warn(
+          `Running debug on schedule: got cache key ${request.response.redisCacheKey}`,
+          { text },
+        );
+      }
+
       return {
         request,
         decodedResponseText: text,
@@ -217,13 +232,20 @@ export class FeedFetcherService {
         request.status = RequestStatus.BAD_STATUS_CODE;
       }
 
+      const etag = res.headers.get('etag');
+      const lastModified = res.headers.get('last-modified');
+
       const response = new Response();
       response.createdAt = request.createdAt;
       response.statusCode = res.status;
       response.headers = {};
 
-      for (const [key, val] of res.headers.entries()) {
-        response.headers[key] = val;
+      if (etag) {
+        response.headers.etag = etag;
+      }
+
+      if (lastModified) {
+        response.headers.lastModified = lastModified;
       }
 
       let text: string | null = null;
@@ -437,11 +459,6 @@ export class FeedFetcherService {
       convertHeaderValue(normalizedHeaders.get('last-modified')),
     );
     headers.set('server', convertHeaderValue(normalizedHeaders.get('server')));
-    headers.set('date', convertHeaderValue(normalizedHeaders.get('date')));
-    headers.set(
-      'cache-control',
-      convertHeaderValue(normalizedHeaders.get('cache-control')),
-    );
 
     return {
       headers,

diff --git a/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts b/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts
@@ -120,19 +120,11 @@ export default class PartitionedRequestsStoreService {
 
   async getLatestOkRequest(
     lookupKey: string,
-    opts?: {
-      fields?: Record<'response_headers', boolean>;
-    },
-  ): Promise<null | {
-    createdAt: Date;
-    responseHeaders?: Record<string, string>;
-  }> {
+  ): Promise<null | { createdAt: Date }> {
     const em = this.orm.em.getConnection();
 
     const [result] = await em.execute(
-      `SELECT created_at ${
-        opts?.fields?.response_headers ? ', response_headers' : ''
-      } FROM request_partitioned
+      `SELECT created_at FROM request_partitioned
        WHERE lookup_key = ?
        AND status = 'OK'
        ORDER BY created_at DESC
@@ -144,10 +136,7 @@ export default class PartitionedRequestsStoreService {
       return null;
     }
 
-    return {
-      createdAt: new Date(result.created_at),
-      responseHeaders: result.response_headers,
-    };
+    return { createdAt: new Date(result.created_at) };
   }
 
   async countFailedRequests(lookupKey: string, since?: Date) {