From 391e217ebe4786d78924b6f78878a5ebc9510700 Mon Sep 17 00:00:00 2001 From: Carson McManus Date: Tue, 12 Mar 2024 15:12:16 -0400 Subject: [PATCH] cache sponsorblock segments in redis (#1482) --- common/constants.ts | 4 +- server/ott-config.ts | 15 +++++-- server/room.ts | 18 +++----- server/sponsorblock.ts | 46 ++++++++++++++++++- server/tests/unit/sponsorblock.spec.ts | 62 ++++++++++++++++++++++++++ 5 files changed, 126 insertions(+), 19 deletions(-) create mode 100644 server/tests/unit/sponsorblock.spec.ts diff --git a/common/constants.ts b/common/constants.ts index 633b3ecc7..08939fb83 100644 --- a/common/constants.ts +++ b/common/constants.ts @@ -1,3 +1,5 @@ +import type { Category } from "sponsorblock-api"; + export const ANNOUNCEMENT_CHANNEL = "announcement"; export const ROOM_NAME_REGEX = /^[a-z0-9_-]+$/i; export const USERNAME_LENGTH_MAX = 48; @@ -14,7 +16,7 @@ export const ALL_VIDEO_SERVICES = [ "peertube", "pluto", ] as const; -export const ALL_SKIP_CATEGORIES = [ +export const ALL_SKIP_CATEGORIES: Category[] = [ "sponsor", "intro", "outro", diff --git a/server/ott-config.ts b/server/ott-config.ts index be0caf5ad..0ebb000a6 100644 --- a/server/ott-config.ts +++ b/server/ott-config.ts @@ -444,10 +444,17 @@ export const conf = convict({ }, }, video: { - enable_sponsorblock: { - doc: "Whether to enable fetching skipable segments from sponsorblock.", - format: Boolean, - default: true, + sponsorblock: { + enabled: { + doc: "Whether to enable fetching skipable segments from sponsorblock.", + format: Boolean, + default: true, + }, + cache_ttl: { + doc: "The duration in seconds to cache sponsorblock segments for.", + format: "nat", + default: 60 * 60 * 24 * 7, // 1 week + }, }, }, }); diff --git a/server/room.ts b/server/room.ts index ca6b14731..7e38b9aa0 100644 --- a/server/room.ts +++ b/server/room.ts @@ -60,7 +60,7 @@ import { import storage from "./storage"; import tokens, { SessionInfo } from "./auth/tokens"; import { OttException } from "ott-common/exceptions"; -import { getSponsorBlock } from "./sponsorblock"; +import { fetchSegments, getSponsorBlock } from "./sponsorblock"; import { ResponseError as SponsorblockResponseError, Segment, Category } from "sponsorblock-api"; import { VideoQueue } from "./videoqueue"; import { Counter } from "prom-client"; @@ -225,7 +225,7 @@ export class Room implements RoomState { _owner: User | null = null; grants: Grants = new Grants(); userRoles: Map>; - _autoSkipSegmentCategories: Array = Array.from(ALL_SKIP_CATEGORIES); + _autoSkipSegmentCategories = Array.from(ALL_SKIP_CATEGORIES); restoreQueueBehavior: BehaviorOption = BehaviorOption.Prompt; _enableVoteSkip: boolean = false; @@ -541,7 +541,7 @@ export class Room implements RoomState { } if ( - conf.get("video.enable_sponsorblock") && + conf.get("video.sponsorblock.enabled") && this.autoSkipSegmentCategories.length > 0 && this.currentSource ) { @@ -766,7 +766,7 @@ export class Room implements RoomState { ); } - if (conf.get("video.enable_sponsorblock") && this.autoSkipSegmentCategories.length > 0) { + if (conf.get("video.sponsorblock.enabled") && this.autoSkipSegmentCategories.length > 0) { if (this.wantSponsorBlock) { this.wantSponsorBlock = false; // Disable this before the request to avoid spamming the sponsorblock if the request takes too long. try { @@ -946,15 +946,7 @@ export class Room implements RoomState { this.log.info( `fetching sponsorblock segments for ${this.currentSource.service}:${this.currentSource.id}` ); - const sponsorBlock = await getSponsorBlock(); - this.videoSegments = await sponsorBlock.getSegments(this.currentSource.id, [ - "sponsor", - "intro", - "outro", - "interaction", - "selfpromo", - "preview", - ]); + this.videoSegments = await fetchSegments(this.currentSource.id); } /** Updates playbackPosition according to the computed value, and resets _playbackStart */ diff --git a/server/sponsorblock.ts b/server/sponsorblock.ts index 92cf6aaac..281b1516f 100644 --- a/server/sponsorblock.ts +++ b/server/sponsorblock.ts @@ -1,20 +1,64 @@ -import { SponsorBlock } from "sponsorblock-api"; +import { SponsorBlock, type Segment } from "sponsorblock-api"; import { redisClient } from "./redisclient"; import { v4 as uuidv4 } from "uuid"; +import { ALL_SKIP_CATEGORIES } from "ott-common"; +import { conf } from "./ott-config"; +import { getLogger } from "./logger"; + +const log = getLogger("sponsorblock"); const SPONSORBLOCK_USERID_KEY = `sponsorblock-userid`; +const SEGMENT_CACHE_PREFIX = "segments"; + +let _cachedUserId: string | null = null; export async function getSponsorBlockUserId(): Promise { + if (_cachedUserId) { + return _cachedUserId; + } let userid = await redisClient.get(SPONSORBLOCK_USERID_KEY); if (!userid) { userid = uuidv4(); await redisClient.set(SPONSORBLOCK_USERID_KEY, userid); } + _cachedUserId = userid; return userid; } +/** Used for tests. */ +export function clearUserId() { + _cachedUserId = null; +} + export async function getSponsorBlock(): Promise { const userid = await getSponsorBlockUserId(); const sponsorblock = new SponsorBlock(userid); return sponsorblock; } + +export async function fetchSegments(videoId: string): Promise { + if (conf.get("video.sponsorblock.cache_ttl") > 0) { + const cachedSegments = await redisClient.get(`${SEGMENT_CACHE_PREFIX}:${videoId}`); + if (cachedSegments) { + try { + return JSON.parse(cachedSegments); + } catch (e) { + log.warn( + `Failed to parse cached segments for video ${videoId}, fetching fresh segments` + ); + } + } + } + const sponsorblock = await getSponsorBlock(); + const segments = await sponsorblock.getSegments(videoId, ALL_SKIP_CATEGORIES); + await cacheSegments(videoId, segments); + return segments; +} + +async function cacheSegments(videoId: string, segments: Segment[]) { + await redisClient.setEx( + `${SEGMENT_CACHE_PREFIX}:${videoId}`, + conf.get("video.sponsorblock.cache_ttl"), + JSON.stringify(segments) + ); +} diff --git a/server/tests/unit/sponsorblock.spec.ts b/server/tests/unit/sponsorblock.spec.ts new file mode 100644 index 000000000..ebbb8af96 --- /dev/null +++ b/server/tests/unit/sponsorblock.spec.ts @@ -0,0 +1,62 @@ +import { describe, it, expect, vi, beforeAll, beforeEach, afterEach } from "vitest"; +import { redisClient, buildClients } from "../../redisclient"; +import * as sponsorblock from "../../sponsorblock"; +import { SponsorBlock } from "sponsorblock-api"; + +vi.mock("sponsorblock-api", () => { + return { + SponsorBlock: class { + getSegments = vi.fn().mockResolvedValue([{ start: 0, end: 1, category: "sponsor" }]); + }, + }; +}); + +describe("SponsorBlock", () => { + beforeAll(async () => { + await buildClients(); + }); + + it("should generate and return the same user id and only ping redis once", async () => { + sponsorblock.clearUserId(); + let getSpy = vi.spyOn(redisClient, "get").mockResolvedValue(null); + let setSpy = vi.spyOn(redisClient, "set"); + const { getSponsorBlockUserId } = sponsorblock; + const userId = await getSponsorBlockUserId(); + expect(userId).toBeDefined(); + const userId2 = await getSponsorBlockUserId(); + expect(userId2).toEqual(userId); + + expect(redisClient.get).toHaveBeenCalledTimes(1); + expect(redisClient.set).toHaveBeenCalledTimes(1); + getSpy.mockRestore(); + setSpy.mockRestore(); + }); + + it("should always return the same user id and only ping redis once", async () => { + sponsorblock.clearUserId(); + let getSpy = vi.spyOn(redisClient, "get").mockResolvedValue("testuserid"); + let setSpy = vi.spyOn(redisClient, "set"); + const { getSponsorBlockUserId } = sponsorblock; + const userId = await getSponsorBlockUserId(); + expect(userId).toBe("testuserid"); + const userId2 = await getSponsorBlockUserId(); + expect(userId2).toBe("testuserid"); + + expect(redisClient.get).toHaveBeenCalledTimes(1); + expect(redisClient.set).not.toHaveBeenCalled(); + getSpy.mockRestore(); + setSpy.mockRestore(); + }); + + it("should fetch new segments if parsing cached entry fails", async () => { + let getSpy = vi.spyOn(redisClient, "get").mockResolvedValue("[invalid json"); + let setSpy = vi.spyOn(redisClient, "setEx"); + const { fetchSegments } = sponsorblock; + const segments = await fetchSegments("testvideo"); + expect(segments).toEqual([{ start: 0, end: 1, category: "sponsor" }]); + expect(redisClient.get).toHaveBeenCalledTimes(1); + expect(redisClient.setEx).toHaveBeenCalledTimes(1); + getSpy.mockRestore(); + setSpy.mockRestore(); + }); +});