diff --git a/services/feed-requests/migrations/Migration20241231120419.ts b/services/feed-requests/migrations/Migration20241231120419.ts deleted file mode 100644 index 54324b424..000000000 --- a/services/feed-requests/migrations/Migration20241231120419.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Migration } from '@mikro-orm/migrations'; - -export class Migration20241231120419 extends Migration { - - async up(): Promise { - this.addSql(`ALTER TYPE request_partitioned_status ADD VALUE 'CACHE_CONTROL_SKIPPED';`); - // switch the created_at and status of the index - this.addSql(`DROP INDEX request_partitioned_lookupkey_created_at_status_index;`); - this.addSql(`CREATE INDEX request_partitioned_lookupkey_status_created_at_index ON request_partitioned (lookup_key, status, created_at);`); - } - - async down(): Promise { - this.addSql(`ALTER TYPE request_partitioned_status DROP VALUE 'CACHE_CONTROL_SKIPPED';`); - // switch the created_at and status of the index - this.addSql(`DROP INDEX request_partitioned_lookupkey_status_created_at_index;`); - this.addSql(`CREATE INDEX request_partitioned_lookupkey_created_at_status_index ON request_partitioned (lookup_key, created_at, status);`); - } - -} diff --git a/services/feed-requests/src/feed-fetcher/entities/response.entity.ts b/services/feed-requests/src/feed-fetcher/entities/response.entity.ts index df765e4eb..abbca7bf3 100644 --- a/services/feed-requests/src/feed-fetcher/entities/response.entity.ts +++ b/services/feed-requests/src/feed-fetcher/entities/response.entity.ts @@ -49,7 +49,10 @@ export class Response { type: 'json', nullable: true, }) - headers?: Record | null; + headers?: { + etag?: string; + lastModified?: string; + } | null; @Property({ type: 'timestamp with time zone', diff --git a/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts b/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts index 7e7c0bd75..5795eafae 100644 --- a/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts +++ b/services/feed-requests/src/feed-fetcher/feed-fetcher.service.ts @@ -58,7 +58,7 @@ interface FetchOptions { interface FetchResponse { ok: boolean; status: number; - headers: Map; + headers: Map<'etag' | 'last-modified' | 'server' | 'content-type', string>; text: () => Promise; arrayBuffer: () => Promise; } @@ -135,11 +135,19 @@ export class FeedFetcherService { request: Request; decodedResponseText: string | null | undefined; } | null> { + const logDebug = + url === + 'https://www.clanaod.net/forums/external.php?type=RSS2&forumids=102'; + const request = await this.partitionedRequestsStore.getLatestRequest( lookupKey || url, ); if (!request) { + if (logDebug) { + logger.warn(`Running debug on schedule: no request was found`); + } + return null; } @@ -154,6 +162,13 @@ export class FeedFetcherService { ).toString() : ''; + if (logDebug) { + logger.warn( + `Running debug on schedule: got cache key ${request.response.redisCacheKey}`, + { text }, + ); + } + return { request, decodedResponseText: text, @@ -217,13 +232,20 @@ export class FeedFetcherService { request.status = RequestStatus.BAD_STATUS_CODE; } + const etag = res.headers.get('etag'); + const lastModified = res.headers.get('last-modified'); + const response = new Response(); response.createdAt = request.createdAt; response.statusCode = res.status; response.headers = {}; - for (const [key, val] of res.headers.entries()) { - response.headers[key] = val; + if (etag) { + response.headers.etag = etag; + } + + if (lastModified) { + response.headers.lastModified = lastModified; } let text: string | null = null; @@ -437,11 +459,6 @@ export class FeedFetcherService { convertHeaderValue(normalizedHeaders.get('last-modified')), ); headers.set('server', convertHeaderValue(normalizedHeaders.get('server'))); - headers.set('date', convertHeaderValue(normalizedHeaders.get('date'))); - headers.set( - 'cache-control', - convertHeaderValue(normalizedHeaders.get('cache-control')), - ); return { headers, diff --git a/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts b/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts index 4938a63d8..c514ef0bf 100644 --- a/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts +++ b/services/feed-requests/src/partitioned-requests-store/partitioned-requests-store.service.ts @@ -120,19 +120,11 @@ export default class PartitionedRequestsStoreService { async getLatestOkRequest( lookupKey: string, - opts?: { - fields?: Record<'response_headers', boolean>; - }, - ): Promise; - }> { + ): Promise { const em = this.orm.em.getConnection(); const [result] = await em.execute( - `SELECT created_at ${ - opts?.fields?.response_headers ? ', response_headers' : '' - } FROM request_partitioned + `SELECT created_at FROM request_partitioned WHERE lookup_key = ? AND status = 'OK' ORDER BY created_at DESC @@ -144,10 +136,7 @@ export default class PartitionedRequestsStoreService { return null; } - return { - createdAt: new Date(result.created_at), - responseHeaders: result.response_headers, - }; + return { createdAt: new Date(result.created_at) }; } async countFailedRequests(lookupKey: string, since?: Date) {