From 4f15900265875c72462d5e4590aac6367db396a2 Mon Sep 17 00:00:00 2001 From: Jacob-Tate Date: Fri, 2 Aug 2024 13:15:52 +0530 Subject: [PATCH 01/61] komga implementation --- apps/backend/src/entities/integration.rs | 1 - apps/backend/src/integrations.rs | 81 ++++++++++++++++++- apps/backend/src/miscellaneous.rs | 52 +++++++++++- apps/backend/src/models.rs | 72 +++++++++++++++++ apps/backend/src/providers/manga_updates.rs | 10 ++- .../_dashboard.settings.integrations.tsx | 55 ++++++++++++- libs/database/src/definitions.rs | 1 + libs/generated/src/graphql/backend/gql.ts | 4 +- libs/generated/src/graphql/backend/graphql.ts | 29 ++++++- libs/graphql/src/backend/queries/combined.gql | 3 + 10 files changed, 298 insertions(+), 10 deletions(-) diff --git a/apps/backend/src/entities/integration.rs b/apps/backend/src/entities/integration.rs index f9aec10cf2..e949cddbd9 100644 --- a/apps/backend/src/entities/integration.rs +++ b/apps/backend/src/entities/integration.rs @@ -27,7 +27,6 @@ pub struct Model { #[graphql(skip_input)] pub last_triggered_on: Option, #[sea_orm(column_type = "Json")] - #[graphql(skip)] pub provider_specifics: Option, } diff --git a/apps/backend/src/integrations.rs b/apps/backend/src/integrations.rs index 41ffb097bf..e071cabc8c 100644 --- a/apps/backend/src/integrations.rs +++ b/apps/backend/src/integrations.rs @@ -27,7 +27,7 @@ use sonarr_api_rs::{ use crate::{ entities::{metadata, prelude::Metadata}, - models::{audiobookshelf_models, media::CommitMediaInput}, + models::{komga_models, audiobookshelf_models, media::CommitMediaInput}, providers::google_books::GoogleBooksService, traits::TraceOk, utils::{get_base_http_client, ilike_sql}, @@ -399,6 +399,85 @@ impl IntegrationService { Ok(payload) } + pub async fn komga_progress( + &self, + base_url: &str, + cookie: &str, + provider: MediaSource + ) -> Result<(Vec, Vec)> + { + let client = get_base_http_client(&format!("{}/api/v1/", base_url), None); + + let response = client + .get("series?read_status=IN_PROGRESS") + .header("Cookie", cookie) + .send() + .await + .map_err(|e| anyhow!(e))? + .json::() + .await.unwrap(); + + let mut media_items = vec![]; + for item in response.content.iter() { + let mut current_chapter = item.books_read_count.unwrap(); + let id; + let source; + let providers = item.metadata.find_providers(); + + if !providers.is_empty() { + (source, id) = match providers.iter() + .find(|x| x.0.unwrap() == provider) + .cloned() { + Some((s, i)) => (s, i), + None => providers.get(0) + .map(|x| x.clone()) + .unwrap_or_else(|| (None, None)) + } + } + else { + let db_manga = Metadata::find() + .filter(metadata::Column::Lot.eq(MediaLot::Manga)) + .filter( + Condition::all() + .add(Expr::col(metadata::Column::Title).eq(&item.name)), + ) + .one(&self.db) + .await?; + + if let Some(manga) = db_manga { + id = Some(manga.identifier.clone()); + source = Some(manga.source.clone()); + } + else { + tracing::debug!("No MAL URL or database entry found for manga: {}", item.name); + + continue; + } + } + + if source.is_some() { + while current_chapter > 0 { + media_items.push(IntegrationMediaSeen { + identifier: id.clone().unwrap(), + lot: MediaLot::Manga, + source: source.unwrap(), + manga_chapter_number: Some(current_chapter), + progress: dec!(100), + provider_watched_on: Some("Komga".to_string()), + ..Default::default() + }); + + current_chapter -= 1; + } + } + else { + tracing::debug!("No MAL URL or database entry found for manga: {}", item.name); + } + } + + Ok((media_items, vec![])) + } + pub async fn audiobookshelf_progress( &self, base_url: &str, diff --git a/apps/backend/src/miscellaneous.rs b/apps/backend/src/miscellaneous.rs index 611eb393f5..0f16e716f4 100644 --- a/apps/backend/src/miscellaneous.rs +++ b/apps/backend/src/miscellaneous.rs @@ -164,6 +164,7 @@ struct UpdateUserIntegrationInput { is_disabled: Option, minimum_progress: Option, maximum_progress: Option, + provider_specifics: Option, } #[derive(Debug, Serialize, Deserialize, InputObject, Clone)] @@ -2519,6 +2520,37 @@ impl MiscellaneousService { None }; let manga_ei = if matches!(meta.lot, MediaLot::Manga) { + if respect_cache { + let test = SeenMangaExtraInformation { + volume: None, + chapter: input.manga_chapter_number + }; + + let all_seen = Seen::find() + .filter(seen::Column::Progress.eq(100)) + .filter(seen::Column::UserId.eq(user_id)) + .filter(seen::Column::State.ne(SeenState::Dropped)) + .filter(seen::Column::MetadataId.eq(&input.metadata_id)) + .filter( + seen::Column::MangaExtraInformation.eq(test) + // Expr::expr(Func::cast_as( + // Expr::col(seen::Column::MangaExtraInformation), + // Alias::new("text") + // )) + // .ilike(ilike_sql(format!("\"chapter\": {}", input.manga_chapter_number.unwrap()).to_string().as_ref())) + ) + .order_by_desc(seen::Column::LastUpdatedOn) + .all(&self.db) + .await + .unwrap(); + + if !all_seen.is_empty() { + return Ok(ProgressUpdateResultUnion::Error(ProgressUpdateError { + error: ProgressUpdateErrorVariant::AlreadySeen, + })); + } + } + Some(SeenMangaExtraInformation { chapter: input.manga_chapter_number, volume: input.manga_volume_number, @@ -5492,6 +5524,7 @@ impl MiscellaneousService { } let lot = match input.provider { IntegrationProvider::Audiobookshelf => IntegrationLot::Yank, + IntegrationProvider::Komga => IntegrationLot::Yank, IntegrationProvider::Radarr | IntegrationProvider::Sonarr => IntegrationLot::Push, _ => IntegrationLot::Sink, }; @@ -5535,6 +5568,13 @@ impl MiscellaneousService { if let Some(d) = input.is_disabled { db_integration.is_disabled = ActiveValue::Set(Some(d)); } + if let Some(d) = input.provider_specifics { + let mut existing_specifics = + db_integration.provider_specifics.clone().unwrap().clone().unwrap().clone(); + + existing_specifics.komga_provider = d.komga_provider; + db_integration.provider_specifics = ActiveValue::Set(Some(existing_specifics)); + } db_integration.update(&self.db).await?; Ok(true) } @@ -5768,7 +5808,17 @@ impl MiscellaneousService { |input| self.commit_metadata(input), ) .await - } + }, + IntegrationProvider::Komga => { + let specifics = integration.clone().provider_specifics.unwrap(); + self.get_integration_service() + .komga_progress( + &specifics.komga_base_url.unwrap(), + &specifics.komga_cookie.unwrap(), + specifics.komga_provider.unwrap(), + ) + .await + }, _ => continue, }; if let Ok((seen_progress, collection_progress)) = response { diff --git a/apps/backend/src/models.rs b/apps/backend/src/models.rs index 1af78bcdc2..190d269166 100644 --- a/apps/backend/src/models.rs +++ b/apps/backend/src/models.rs @@ -1311,6 +1311,9 @@ pub mod media { pub plex_username: Option, pub audiobookshelf_base_url: Option, pub audiobookshelf_token: Option, + pub komga_base_url: Option, + pub komga_cookie: Option, + pub komga_provider: Option, pub radarr_base_url: Option, pub radarr_api_key: Option, pub radarr_profile_id: Option, @@ -1904,6 +1907,75 @@ pub mod importer { } } +pub mod komga_models { + use openidconnect::url::Url; + use super::*; + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Link { + pub label: String, + pub url: String, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Metadata { + pub links: Vec, + } + + impl Metadata { + fn extract_id(&self, url: String) -> Option { + if let Ok(parsed_url) = Url::parse(&url) { + parsed_url.path_segments() + .and_then(|segments| segments.collect::>().get(1).cloned()) + .map(String::from) + } else { + None + } + } + + pub fn find_providers(&self) -> Vec<(Option,Option)> { + let mut provider_links = vec![]; + for link in self.links.iter() { + let source; + + // NOTE: mangaupdates doesnt work here because the ID isnt in the url + match link.label.to_lowercase().as_str() { + "anilist" => source = Some(MediaSource::Anilist), + "myanimelist" => source = Some(MediaSource::Mal), + _ => continue + } + + if source.is_some() { + let id = self.extract_id(link.url.clone()); + provider_links.push((source, id)); + } + } + + provider_links.sort_by_key(|a| a.1.clone()); + provider_links + } + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Item { + pub id: String, + pub name: String, + pub books_count: Decimal, + pub books_read_count: Option, + pub books_unread_count: Decimal, + pub metadata: Metadata, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Response { + pub content: Vec, + } +} + pub mod audiobookshelf_models { use super::*; diff --git a/apps/backend/src/providers/manga_updates.rs b/apps/backend/src/providers/manga_updates.rs index fde820f470..fab4792093 100644 --- a/apps/backend/src/providers/manga_updates.rs +++ b/apps/backend/src/providers/manga_updates.rs @@ -147,7 +147,13 @@ struct MetadataSearchResponse { } impl MangaUpdatesService { - fn extract_status(&self, input: &str) -> (Option, Option) { + fn extract_status(&self, input: Option) -> (Option, Option) { + if input.is_none() { + return (None, None) + } + + let input = input.unwrap(); + let first_part = input.split("
").next().unwrap_or("").trim(); let parts: Vec<&str> = first_part.split_whitespace().collect(); @@ -330,7 +336,7 @@ impl MediaProvider for MangaUpdatesService { } } - let (volumes, status) = self.extract_status(&data.status.clone().unwrap()); + let (volumes, status) = self.extract_status(data.status.clone()); Ok(MediaDetails { identifier: data.series_id.unwrap().to_string(), diff --git a/apps/frontend/app/routes/_dashboard.settings.integrations.tsx b/apps/frontend/app/routes/_dashboard.settings.integrations.tsx index 81dd33c04c..10564bccec 100644 --- a/apps/frontend/app/routes/_dashboard.settings.integrations.tsx +++ b/apps/frontend/app/routes/_dashboard.settings.integrations.tsx @@ -29,6 +29,7 @@ import { DeleteUserIntegrationDocument, GenerateAuthTokenDocument, IntegrationProvider, + MediaSource, UpdateUserIntegrationDocument, UserIntegrationsDocument, type UserIntegrationsQuery, @@ -53,7 +54,10 @@ import { dayjsLib } from "~/lib/generals"; import { useConfirmSubmit, useUserCollections } from "~/lib/hooks"; import { createToastHeaders, serverGqlService } from "~/lib/utilities.server"; -const YANK_INTEGRATIONS = [IntegrationProvider.Audiobookshelf]; +const YANK_INTEGRATIONS = [ + IntegrationProvider.Audiobookshelf, + IntegrationProvider.Komga +]; const PUSH_INTEGRATIONS = [ IntegrationProvider.Radarr, IntegrationProvider.Sonarr, @@ -159,6 +163,9 @@ const createSchema = z.object({ plexUsername: z.string().optional(), audiobookshelfBaseUrl: z.string().optional(), audiobookshelfToken: z.string().optional(), + komgaBaseUrl: z.string().optional(), + komgaCookie: z.string().optional(), + komgaProvider: z.nativeEnum(MediaSource).optional(), radarrBaseUrl: z.string().optional(), radarrApiKey: z.string().optional(), radarrProfileId: z.number().optional(), @@ -182,6 +189,11 @@ const updateSchema = z.object({ minimumProgress: z.string().optional(), maximumProgress: z.string().optional(), isDisabled: zx.CheckboxAsString.optional(), + providerSpecifics: z + .object({ + komgaProvider: z.nativeEnum(MediaSource), + }) + .optional(), }); export default function Page() { @@ -443,6 +455,29 @@ const CreateIntegrationModal = (props: { /> )) + .with(IntegrationProvider.Komga, () => ( + <> + + + ({ + label: changeCase(is), + value: is, + }))} + /> + + )) + .otherwise(() => undefined)} ; minimumProgress?: Maybe; provider: IntegrationProvider; + providerSpecifics?: Maybe; }; export enum IntegrationLot { @@ -716,14 +717,37 @@ export enum IntegrationProvider { Emby = 'EMBY', Jellyfin = 'JELLYFIN', Kodi = 'KODI', + Komga = 'KOMGA', Plex = 'PLEX', Radarr = 'RADARR', Sonarr = 'SONARR' } +export type IntegrationProviderSpecifics = { + audiobookshelfBaseUrl?: Maybe; + audiobookshelfToken?: Maybe; + komgaBaseUrl?: Maybe; + komgaCookie?: Maybe; + komgaProvider?: Maybe; + plexUsername?: Maybe; + radarrApiKey?: Maybe; + radarrBaseUrl?: Maybe; + radarrProfileId?: Maybe; + radarrRootFolderPath?: Maybe; + radarrSyncCollectionIds?: Maybe>; + sonarrApiKey?: Maybe; + sonarrBaseUrl?: Maybe; + sonarrProfileId?: Maybe; + sonarrRootFolderPath?: Maybe; + sonarrSyncCollectionIds?: Maybe>; +}; + export type IntegrationSourceSpecificsInput = { audiobookshelfBaseUrl?: InputMaybe; audiobookshelfToken?: InputMaybe; + komgaBaseUrl?: InputMaybe; + komgaCookie?: InputMaybe; + komgaProvider?: InputMaybe; plexUsername?: InputMaybe; radarrApiKey?: InputMaybe; radarrBaseUrl?: InputMaybe; @@ -1906,6 +1930,7 @@ export type UpdateUserIntegrationInput = { isDisabled?: InputMaybe; maximumProgress?: InputMaybe; minimumProgress?: InputMaybe; + providerSpecifics?: InputMaybe; }; export type UpdateUserNotificationPlatformInput = { @@ -2961,7 +2986,7 @@ export type UserCollectionsListQuery = { userCollectionsList: Array<{ id: string export type UserIntegrationsQueryVariables = Exact<{ [key: string]: never; }>; -export type UserIntegrationsQuery = { userIntegrations: Array<{ id: string, lot: IntegrationLot, provider: IntegrationProvider, createdOn: string, isDisabled?: boolean | null, maximumProgress?: string | null, minimumProgress?: string | null, lastTriggeredOn?: string | null }> }; +export type UserIntegrationsQuery = { userIntegrations: Array<{ id: string, lot: IntegrationLot, provider: IntegrationProvider, createdOn: string, isDisabled?: boolean | null, maximumProgress?: string | null, minimumProgress?: string | null, lastTriggeredOn?: string | null, providerSpecifics?: { komgaProvider?: MediaSource | null } | null }> }; export type UserNotificationPlatformsQueryVariables = Exact<{ [key: string]: never; }>; @@ -3126,7 +3151,7 @@ export const GetPresignedS3UrlDocument = {"kind":"Document","definitions":[{"kin export const ProvidersLanguageInformationDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"ProvidersLanguageInformation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"providersLanguageInformation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"supported"}},{"kind":"Field","name":{"kind":"Name","value":"default"}},{"kind":"Field","name":{"kind":"Name","value":"source"}}]}}]}}]} as unknown as DocumentNode; export const UserExportsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"UserExports"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"userExports"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"startedAt"}},{"kind":"Field","name":{"kind":"Name","value":"endedAt"}},{"kind":"Field","name":{"kind":"Name","value":"url"}},{"kind":"Field","name":{"kind":"Name","value":"exported"}}]}}]}}]} as unknown as DocumentNode; export const UserCollectionsListDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"UserCollectionsList"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"name"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"userCollectionsList"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"name"},"value":{"kind":"Variable","name":{"kind":"Name","value":"name"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"count"}},{"kind":"Field","name":{"kind":"Name","value":"isDefault"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"creator"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"collaborators"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"name"}}]}},{"kind":"Field","name":{"kind":"Name","value":"informationTemplate"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"lot"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"required"}},{"kind":"Field","name":{"kind":"Name","value":"description"}},{"kind":"Field","name":{"kind":"Name","value":"defaultValue"}}]}}]}}]}}]} as unknown as DocumentNode; -export const UserIntegrationsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"UserIntegrations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"userIntegrations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"lot"}},{"kind":"Field","name":{"kind":"Name","value":"provider"}},{"kind":"Field","name":{"kind":"Name","value":"createdOn"}},{"kind":"Field","name":{"kind":"Name","value":"isDisabled"}},{"kind":"Field","name":{"kind":"Name","value":"maximumProgress"}},{"kind":"Field","name":{"kind":"Name","value":"minimumProgress"}},{"kind":"Field","name":{"kind":"Name","value":"lastTriggeredOn"}}]}}]}}]} as unknown as DocumentNode; +export const UserIntegrationsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"UserIntegrations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"userIntegrations"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"lot"}},{"kind":"Field","name":{"kind":"Name","value":"provider"}},{"kind":"Field","name":{"kind":"Name","value":"createdOn"}},{"kind":"Field","name":{"kind":"Name","value":"isDisabled"}},{"kind":"Field","name":{"kind":"Name","value":"maximumProgress"}},{"kind":"Field","name":{"kind":"Name","value":"minimumProgress"}},{"kind":"Field","name":{"kind":"Name","value":"lastTriggeredOn"}},{"kind":"Field","name":{"kind":"Name","value":"providerSpecifics"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"komgaProvider"}}]}}]}}]}}]} as unknown as DocumentNode; export const UserNotificationPlatformsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"UserNotificationPlatforms"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"userNotificationPlatforms"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"lot"}},{"kind":"Field","name":{"kind":"Name","value":"createdOn"}},{"kind":"Field","name":{"kind":"Name","value":"isDisabled"}},{"kind":"Field","name":{"kind":"Name","value":"description"}}]}}]}}]} as unknown as DocumentNode; export const UsersListDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"UsersList"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"query"}},"type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"usersList"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"query"},"value":{"kind":"Variable","name":{"kind":"Name","value":"query"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"lot"}},{"kind":"Field","name":{"kind":"Name","value":"name"}},{"kind":"Field","name":{"kind":"Name","value":"isDisabled"}}]}}]}}]} as unknown as DocumentNode; export const UserUpcomingCalendarEventsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"UserUpcomingCalendarEvents"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"UserUpcomingCalendarEventInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"userUpcomingCalendarEvents"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"CalendarEventPart"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"SeenShowExtraInformationPart"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"SeenShowExtraInformation"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"episode"}},{"kind":"Field","name":{"kind":"Name","value":"season"}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"SeenPodcastExtraInformationPart"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"SeenPodcastExtraInformation"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"episode"}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"CalendarEventPart"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlCalendarEvent"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"date"}},{"kind":"Field","name":{"kind":"Name","value":"metadataId"}},{"kind":"Field","name":{"kind":"Name","value":"metadataLot"}},{"kind":"Field","name":{"kind":"Name","value":"episodeName"}},{"kind":"Field","name":{"kind":"Name","value":"metadataTitle"}},{"kind":"Field","name":{"kind":"Name","value":"metadataImage"}},{"kind":"Field","name":{"kind":"Name","value":"calendarEventId"}},{"kind":"Field","name":{"kind":"Name","value":"showExtraInformation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"SeenShowExtraInformationPart"}}]}},{"kind":"Field","name":{"kind":"Name","value":"podcastExtraInformation"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"SeenPodcastExtraInformationPart"}}]}}]}}]} as unknown as DocumentNode; diff --git a/libs/graphql/src/backend/queries/combined.gql b/libs/graphql/src/backend/queries/combined.gql index 9c150c7ea8..e3d5b7bf63 100644 --- a/libs/graphql/src/backend/queries/combined.gql +++ b/libs/graphql/src/backend/queries/combined.gql @@ -69,6 +69,9 @@ query UserIntegrations { maximumProgress minimumProgress lastTriggeredOn + providerSpecifics { + komgaProvider + } } } From b5b39cd7927ba386a825ff733d8e657ba463d33f Mon Sep 17 00:00:00 2001 From: Jacob-Tate Date: Sun, 4 Aug 2024 15:11:39 +0530 Subject: [PATCH 02/61] Updated Komga integration to use SSE --- Cargo.lock | 12 + apps/backend/Cargo.toml | 1 + apps/backend/src/background.rs | 6 + apps/backend/src/integrations.rs | 83 +---- apps/backend/src/integrations/komga.rs | 333 ++++++++++++++++++ apps/backend/src/miscellaneous.rs | 133 ++++++- apps/backend/src/models.rs | 67 +--- .../_dashboard.settings.integrations.tsx | 6 +- libs/database/src/definitions.rs | 1 + libs/generated/src/graphql/backend/graphql.ts | 1 + 10 files changed, 486 insertions(+), 157 deletions(-) create mode 100644 apps/backend/src/integrations/komga.rs diff --git a/Cargo.lock b/Cargo.lock index 71ced1155e..cf343380f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1935,6 +1935,17 @@ version = "2.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" +[[package]] +name = "eventsource-stream" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" +dependencies = [ + "futures-core", + "nom", + "pin-project-lite", +] + [[package]] name = "fast_chemail" version = "0.9.6" @@ -4519,6 +4530,7 @@ dependencies = [ "dotenvy_macro", "educe 0.6.0", "enum_meta", + "eventsource-stream", "flate2", "futures", "graphql_client", diff --git a/apps/backend/Cargo.toml b/apps/backend/Cargo.toml index 90d8c69093..8c15a5176d 100644 --- a/apps/backend/Cargo.toml +++ b/apps/backend/Cargo.toml @@ -82,3 +82,4 @@ tracing = { workspace = true } tracing-subscriber = "=0.3.18" logs-wheel = "=0.3.1" uuid = { version = "=1.10.0", features = ["v4"], default-features = false } +eventsource-stream = "0.2.3" diff --git a/apps/backend/src/background.rs b/apps/backend/src/background.rs index 80960ccbc8..26bb7f3f6b 100644 --- a/apps/backend/src/background.rs +++ b/apps/backend/src/background.rs @@ -46,6 +46,8 @@ pub async fn sync_integrations_data( _information: ScheduledJob, misc_service: Data>, ) -> Result<(), Error> { + tracing::trace!("Getting data from sse integrations for all users"); + misc_service.sse_integrations_data().await.unwrap(); tracing::trace!("Getting data from yanked integrations for all users"); misc_service.yank_integrations_data().await.unwrap(); tracing::trace!("Sending data for push integrations for all users"); @@ -85,6 +87,10 @@ pub async fn perform_core_application_job( misc_service .yank_integrations_data_for_user(&user_id) .await + .ok(); + misc_service + .sse_integrations_data_for_user(&user_id) + .await .is_ok() } CoreApplicationJob::BulkProgressUpdate(user_id, input) => misc_service diff --git a/apps/backend/src/integrations.rs b/apps/backend/src/integrations.rs index e071cabc8c..a9b9d0c1b6 100644 --- a/apps/backend/src/integrations.rs +++ b/apps/backend/src/integrations.rs @@ -1,3 +1,5 @@ +mod komga; + use std::future::Future; use anyhow::{anyhow, bail, Result}; @@ -27,7 +29,7 @@ use sonarr_api_rs::{ use crate::{ entities::{metadata, prelude::Metadata}, - models::{komga_models, audiobookshelf_models, media::CommitMediaInput}, + models::{audiobookshelf_models, media::CommitMediaInput}, providers::google_books::GoogleBooksService, traits::TraceOk, utils::{get_base_http_client, ilike_sql}, @@ -399,85 +401,6 @@ impl IntegrationService { Ok(payload) } - pub async fn komga_progress( - &self, - base_url: &str, - cookie: &str, - provider: MediaSource - ) -> Result<(Vec, Vec)> - { - let client = get_base_http_client(&format!("{}/api/v1/", base_url), None); - - let response = client - .get("series?read_status=IN_PROGRESS") - .header("Cookie", cookie) - .send() - .await - .map_err(|e| anyhow!(e))? - .json::() - .await.unwrap(); - - let mut media_items = vec![]; - for item in response.content.iter() { - let mut current_chapter = item.books_read_count.unwrap(); - let id; - let source; - let providers = item.metadata.find_providers(); - - if !providers.is_empty() { - (source, id) = match providers.iter() - .find(|x| x.0.unwrap() == provider) - .cloned() { - Some((s, i)) => (s, i), - None => providers.get(0) - .map(|x| x.clone()) - .unwrap_or_else(|| (None, None)) - } - } - else { - let db_manga = Metadata::find() - .filter(metadata::Column::Lot.eq(MediaLot::Manga)) - .filter( - Condition::all() - .add(Expr::col(metadata::Column::Title).eq(&item.name)), - ) - .one(&self.db) - .await?; - - if let Some(manga) = db_manga { - id = Some(manga.identifier.clone()); - source = Some(manga.source.clone()); - } - else { - tracing::debug!("No MAL URL or database entry found for manga: {}", item.name); - - continue; - } - } - - if source.is_some() { - while current_chapter > 0 { - media_items.push(IntegrationMediaSeen { - identifier: id.clone().unwrap(), - lot: MediaLot::Manga, - source: source.unwrap(), - manga_chapter_number: Some(current_chapter), - progress: dec!(100), - provider_watched_on: Some("Komga".to_string()), - ..Default::default() - }); - - current_chapter -= 1; - } - } - else { - tracing::debug!("No MAL URL or database entry found for manga: {}", item.name); - } - } - - Ok((media_items, vec![])) - } - pub async fn audiobookshelf_progress( &self, base_url: &str, diff --git a/apps/backend/src/integrations/komga.rs b/apps/backend/src/integrations/komga.rs new file mode 100644 index 0000000000..0ef6fba410 --- /dev/null +++ b/apps/backend/src/integrations/komga.rs @@ -0,0 +1,333 @@ +use anyhow::{anyhow, Result, Context}; +use futures::StreamExt; +use rust_decimal::Decimal; +use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, ColumnTrait}; +use sea_query::Expr; +use tokio::sync::{mpsc, mpsc::error::TryRecvError}; +use database::{MediaLot, MediaSource, }; +use eventsource_stream::Eventsource; +use rust_decimal::prelude::{FromPrimitive, Zero}; +use super::{IntegrationMediaCollection, IntegrationMediaSeen, IntegrationService}; +use crate::{ + miscellaneous::SSEObjects, + entities::{metadata, prelude::Metadata}, + utils::{get_base_http_client, }, + models::komga_events +}; + +mod komga_book { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Link { + pub label: String, + pub url: String, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Media { + pub pages_count: i32 + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Metadata { + pub links: Vec, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct ReadProgress { + pub page: i32, + pub completed: bool, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Item { + pub id: String, + pub name: String, + pub series_id: String, + pub media: Media, + pub number: i32, + pub read_progress: ReadProgress, + } +} + +mod komga_series { + use openidconnect::url::Url; + use serde::{Deserialize, Serialize}; + use super::*; + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Link { + pub label: String, + pub url: String, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Metadata { + pub links: Vec, + } + + impl Metadata { + fn extract_id(&self, url: String) -> Option { + if let Ok(parsed_url) = Url::parse(&url) { + parsed_url.path_segments() + .and_then(|segments| segments.collect::>().get(1).cloned()) + .map(String::from) + } else { + None + } + } + + pub fn find_providers(&self) -> Vec<(Option,Option)> { + let mut provider_links = vec![]; + for link in self.links.iter() { + let source; + + // NOTE: mangaupdates doesnt work here because the ID isnt in the url + match link.label.to_lowercase().as_str() { + "anilist" => source = Some(MediaSource::Anilist), + "myanimelist" => source = Some(MediaSource::Mal), + _ => continue + } + + if source.is_some() { + let id = self.extract_id(link.url.clone()); + provider_links.push((source, id)); + } + } + + provider_links.sort_by_key(|a| a.1.clone()); + provider_links + } + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Item { + pub id: String, + pub name: String, + // pub number: Decimal, + pub books_count: Decimal, + pub books_read_count: Option, + pub books_unread_count: Decimal, + pub metadata: Metadata, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Response { + pub content: Vec, + } +} + +impl IntegrationService { + + async fn sse_listener(sender: mpsc::Sender, + base_url: String, + cookie: String,) -> anyhow::Result<(), Box> { + let client = get_base_http_client(&format!("{}/sse/v1/", base_url), None); + + loop { + let response = client + .get("events") + .header("Cookie", cookie.clone()) + .send() + .await + .context("Failed to send request")?; + + let mut stream = response.bytes_stream().eventsource(); + + while let Some(event) = stream.next().await { + let event = event.context("Failed to get next event")?; + tracing::trace!(?event, "Received SSE event"); + + //TODO: Handle the deleted + if event.event == "ReadProgressChanged" { + match serde_json::from_str::(&event.data) { + Ok(read_progress) => { + if sender.send(read_progress).await.is_err() { + tracing::debug!("Receiver dropped, exiting SSE listener"); + break; + } + } + Err(e) => { + tracing::warn!(error = ?e, data = ?event.data, + "Failed to parse ReadProgressChanged event data"); + } + } + } else { + tracing::trace!(event_type = ?event.event, "Received unhandled event type"); + } + } + + tracing::trace!("SSE listener finished"); + tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + } + } + + async fn fetch_book(client: &reqwest::Client, + book_id: &str, + cookie: &str) -> Result { + client + .get(format!("books/{}", book_id)) + .header("Cookie", cookie) + .send() + .await? + .error_for_status()? + .json::() + .await + .map_err(|e| anyhow!("Failed to parse book JSON: {}", e)) + } + + async fn fetch_series(client: &reqwest::Client, + series_id: &str, + cookie: &str) -> Result { + client + .get(format!("series/{}", series_id)) + .header("Cookie", cookie) + .send() + .await? + .error_for_status()? + .json::() + .await + .map_err(|e| anyhow!("Failed to parse series JSON: {}", e)) + } + + + async fn find_provider_and_id( + series: &komga_series::Item, + provider: MediaSource, + db: &DatabaseConnection + ) -> Result<(Option, Option)> { + let providers = series.metadata.find_providers(); + if !providers.is_empty() { + Ok(providers + .iter() + .find(|x| x.0.unwrap() == provider) + .cloned() + .or_else(|| providers.first().cloned()) + .unwrap_or((None, None))) + } else { + let db_manga = Metadata::find() + .filter(metadata::Column::Lot.eq(MediaLot::Manga)) + .filter(Expr::col(metadata::Column::Title).eq(&series.name)) + .one(db) + .await?; + + Ok(db_manga + .map(|manga| (Some(manga.source), Some(manga.identifier))) + .unwrap_or((None, None))) + } + } + + fn calculate_percentage(curr_page: i32, total_page: i32) -> Decimal { + if total_page == 0 { + return Decimal::zero(); // Handle division by zero case + } + + // Perform the calculation using floating-point arithmetic + let percentage = (curr_page as f64 / total_page as f64) * 100.0; + + // Convert the result to Decimal + Decimal::from_f64(percentage).unwrap_or(Decimal::zero()) + } + async fn process_events ( + &self, + base_url: &str, + cookie: &str, + provider: MediaSource, + data: komga_events::Data + ) -> Option { + let client = get_base_http_client(&format!("{}/api/v1/", base_url), None); + + // Fetch book and series data + let book = + IntegrationService::fetch_book(&client, &data.book_id, cookie).await.ok()?; + let series = + IntegrationService::fetch_series(&client, &book.series_id, cookie).await.ok()?; + + // Find provider and ID + let (source, id) = + IntegrationService::find_provider_and_id(&series, provider, &self.db).await.ok()?; + + // If no ID is found, return None + let Some(id) = id else { + tracing::debug!("No MAL URL or database entry found for manga: {}", series.name); + return None; + }; + + Some(IntegrationMediaSeen { + identifier: id, + lot: MediaLot::Manga, + source: source.unwrap(), + manga_chapter_number: Some(book.number), + progress: IntegrationService::calculate_percentage(book.read_progress.page, + book.media.pages_count), + provider_watched_on: Some("Komga".to_string()), + ..Default::default() + }) + } + + pub async fn komga_progress( + &self, + base_url: &str, + cookie: &str, + provider: MediaSource, + sse_lists: &SSEObjects + ) -> Result<(Vec, Vec)> { + let mutex_receiver = sse_lists.get_komga_receiver(); + let mut receiver = { + let mut guard = mutex_receiver.lock().unwrap(); + guard.take() + }; + + if receiver.is_none() { + let (tx, rx) = mpsc::channel::(1000); + receiver = Some(rx); + + let base_url = base_url.to_string(); + let cookie = cookie.to_string(); + let mutex_task = sse_lists.get_komga_task(); + + mutex_task.get_or_init(|| { + tokio::spawn(async move { + if let Err(e) = + IntegrationService::sse_listener(tx, base_url, cookie).await { + tracing::error!("SSE listener error: {}", e); + } + }); + }); + } + + let mut media_items = vec![]; + + if let Some(mut recv) = receiver { + loop { + match recv.try_recv() { + Ok(event) => { + tracing::debug!("Received event {:?}", event); + match self.process_events(base_url, cookie, provider, event).await { + Some(processed_event) => + media_items.push(processed_event), + None => tracing::warn!("Failed to process event"), + } + }, + Err(TryRecvError::Empty) => break, + Err(e) => return Err(anyhow::anyhow!("Receiver error: {}", e)), + } + } + + // Put the receiver back + mutex_receiver.lock().unwrap().replace(recv); + } + + Ok((media_items, vec![])) + } +} \ No newline at end of file diff --git a/apps/backend/src/miscellaneous.rs b/apps/backend/src/miscellaneous.rs index 0f16e716f4..253931bd8e 100644 --- a/apps/backend/src/miscellaneous.rs +++ b/apps/backend/src/miscellaneous.rs @@ -52,7 +52,7 @@ use sea_query::{ }; use serde::{Deserialize, Serialize}; use struson::writer::{JsonStreamWriter, JsonWriter}; - +use tokio::sync::mpsc::Receiver; use crate::{ background::{ApplicationJob, CoreApplicationJob}, entities::{ @@ -95,6 +95,7 @@ use crate::{ CollectionToEntitySystemInformation, DefaultCollection, IdAndNamedObject, MediaStateChanged, SearchDetails, SearchInput, SearchResults, StoredUrl, StringIdObject, UserSummaryData, + komga_events }, providers::{ anilist::{ @@ -125,6 +126,7 @@ use crate::{ user_id_from_token, AUTHOR, SHOW_SPECIAL_SEASON_NAMES, TEMP_DIR, VERSION, }, }; +use std::sync::{Mutex, OnceLock}; type Provider = Box<(dyn MediaProvider + Send + Sync)>; @@ -1473,6 +1475,30 @@ impl MiscellaneousMutation { } } +pub struct SSEObjects { + komga_task: OnceLock<()>, + komga_receiver: Mutex>>, +} + +impl SSEObjects { + const fn new() -> SSEObjects { + SSEObjects { + komga_task: OnceLock::new(), + komga_receiver: Mutex::new(None) + } + } + + pub fn get_komga_receiver(&self) -> &Mutex>> { + &self.komga_receiver + } + + pub fn get_komga_task(&self) -> &OnceLock<()> { + &self.komga_task + } +} + +static SSE_LISTS: SSEObjects = SSEObjects::new(); + pub struct MiscellaneousService { pub db: DatabaseConnection, pub perform_application_job: MemoryStorage, @@ -5524,7 +5550,7 @@ impl MiscellaneousService { } let lot = match input.provider { IntegrationProvider::Audiobookshelf => IntegrationLot::Yank, - IntegrationProvider::Komga => IntegrationLot::Yank, + IntegrationProvider::Komga => IntegrationLot::SSE, IntegrationProvider::Radarr | IntegrationProvider::Sonarr => IntegrationLot::Push, _ => IntegrationLot::Sink, }; @@ -5779,6 +5805,84 @@ impl MiscellaneousService { .collect() } + pub async fn sse_integrations_data_for_user(&self, user_id: &String) -> Result { + let preferences = self.user_preferences(user_id).await?; + if preferences.general.disable_integrations { + return Ok(false); + } + let integrations = Integration::find() + .filter(integration::Column::UserId.eq(user_id)) + .all(&self.db) + .await?; + let mut progress_updates = vec![]; + let mut collection_updates = vec![]; + let mut to_update_integrations = vec![]; + let integration_service = self.get_integration_service(); + for integration in integrations.into_iter() { + if integration.is_disabled.unwrap_or_default() { + tracing::debug!("Integration {} is disabled", integration.id); + continue; + } + let response = match integration.provider { + IntegrationProvider::Komga => { + let specifics = integration.clone().provider_specifics.unwrap(); + + integration_service + .komga_progress( + &specifics.komga_base_url.unwrap(), + &specifics.komga_cookie.unwrap(), + specifics.komga_provider.unwrap(), + &SSE_LISTS + ) + .await + }, + _ => continue, + }; + if let Ok((seen_progress, collection_progress)) = response { + collection_updates.extend(collection_progress); + to_update_integrations.push(integration.id.clone()); + progress_updates.push((integration, seen_progress)); + } + } + for (integration, progress_updates) in progress_updates.into_iter() { + for pu in progress_updates.into_iter() { + self.integration_progress_update(&integration, pu, user_id) + .await + .trace_ok(); + } + } + for col_update in collection_updates.into_iter() { + let metadata::Model { id, .. } = self + .commit_metadata(CommitMediaInput { + lot: col_update.lot, + source: col_update.source, + identifier: col_update.identifier.clone(), + force_update: None, + }) + .await?; + self.add_entity_to_collection( + user_id, + ChangeCollectionToEntityInput { + creator_user_id: user_id.to_owned(), + collection_name: col_update.collection, + metadata_id: Some(id.clone()), + ..Default::default() + }, + ) + .await + .trace_ok(); + } + Integration::update_many() + .filter(integration::Column::Id.is_in(to_update_integrations)) + .col_expr( + integration::Column::LastTriggeredOn, + Expr::value(Utc::now()), + ) + .exec(&self.db) + .await?; + Ok(true) + } + pub async fn yank_integrations_data_for_user(&self, user_id: &String) -> Result { let preferences = self.user_preferences(user_id).await?; if preferences.general.disable_integrations { @@ -5809,16 +5913,6 @@ impl MiscellaneousService { ) .await }, - IntegrationProvider::Komga => { - let specifics = integration.clone().provider_specifics.unwrap(); - self.get_integration_service() - .komga_progress( - &specifics.komga_base_url.unwrap(), - &specifics.komga_cookie.unwrap(), - specifics.komga_provider.unwrap(), - ) - .await - }, _ => continue, }; if let Ok((seen_progress, collection_progress)) = response { @@ -5881,6 +5975,21 @@ impl MiscellaneousService { Ok(()) } + pub async fn sse_integrations_data(&self) -> Result<()> { + let users_with_integrations = Integration::find() + .filter(integration::Column::Lot.eq(IntegrationLot::SSE)) + .select_only() + .column(integration::Column::UserId) + .into_tuple::() + .all(&self.db) + .await?; + for user_id in users_with_integrations { + tracing::debug!("sse integrations data for user {}", user_id); + self.sse_integrations_data_for_user(&user_id).await?; + } + Ok(()) + } + pub async fn send_data_for_push_integrations(&self) -> Result<()> { let users_with_integrations = Integration::find() .filter(integration::Column::Lot.eq(IntegrationLot::Push)) diff --git a/apps/backend/src/models.rs b/apps/backend/src/models.rs index 190d269166..72abf980da 100644 --- a/apps/backend/src/models.rs +++ b/apps/backend/src/models.rs @@ -1907,75 +1907,16 @@ pub mod importer { } } -pub mod komga_models { - use openidconnect::url::Url; +pub mod komga_events { use super::*; #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] - pub struct Link { - pub label: String, - pub url: String, - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Metadata { - pub links: Vec, - } - - impl Metadata { - fn extract_id(&self, url: String) -> Option { - if let Ok(parsed_url) = Url::parse(&url) { - parsed_url.path_segments() - .and_then(|segments| segments.collect::>().get(1).cloned()) - .map(String::from) - } else { - None - } - } - - pub fn find_providers(&self) -> Vec<(Option,Option)> { - let mut provider_links = vec![]; - for link in self.links.iter() { - let source; - - // NOTE: mangaupdates doesnt work here because the ID isnt in the url - match link.label.to_lowercase().as_str() { - "anilist" => source = Some(MediaSource::Anilist), - "myanimelist" => source = Some(MediaSource::Mal), - _ => continue - } - - if source.is_some() { - let id = self.extract_id(link.url.clone()); - provider_links.push((source, id)); - } - } - - provider_links.sort_by_key(|a| a.1.clone()); - provider_links - } - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Item { - pub id: String, - pub name: String, - pub books_count: Decimal, - pub books_read_count: Option, - pub books_unread_count: Decimal, - pub metadata: Metadata, - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Response { - pub content: Vec, + pub struct Data { + pub book_id: String, + pub user_id: String, } } - pub mod audiobookshelf_models { use super::*; diff --git a/apps/frontend/app/routes/_dashboard.settings.integrations.tsx b/apps/frontend/app/routes/_dashboard.settings.integrations.tsx index 10564bccec..0717164021 100644 --- a/apps/frontend/app/routes/_dashboard.settings.integrations.tsx +++ b/apps/frontend/app/routes/_dashboard.settings.integrations.tsx @@ -54,15 +54,17 @@ import { dayjsLib } from "~/lib/generals"; import { useConfirmSubmit, useUserCollections } from "~/lib/hooks"; import { createToastHeaders, serverGqlService } from "~/lib/utilities.server"; +const SSE_INTEGRATION = [ + IntegrationProvider.Komga, +] const YANK_INTEGRATIONS = [ IntegrationProvider.Audiobookshelf, - IntegrationProvider.Komga ]; const PUSH_INTEGRATIONS = [ IntegrationProvider.Radarr, IntegrationProvider.Sonarr, ]; -const NO_SHOW_URL = [...YANK_INTEGRATIONS, ...PUSH_INTEGRATIONS]; +const NO_SHOW_URL = [...SSE_INTEGRATION, ...YANK_INTEGRATIONS, ...PUSH_INTEGRATIONS]; export const loader = unstable_defineLoader(async ({ request }) => { const [{ userIntegrations }] = await Promise.all([ diff --git a/libs/database/src/definitions.rs b/libs/database/src/definitions.rs index 1064624879..ad860abb7a 100644 --- a/libs/database/src/definitions.rs +++ b/libs/database/src/definitions.rs @@ -424,6 +424,7 @@ pub enum IntegrationLot { Yank, Sink, Push, + SSE } #[derive( diff --git a/libs/generated/src/graphql/backend/graphql.ts b/libs/generated/src/graphql/backend/graphql.ts index ca39b8204b..7f95e730e2 100644 --- a/libs/generated/src/graphql/backend/graphql.ts +++ b/libs/generated/src/graphql/backend/graphql.ts @@ -709,6 +709,7 @@ export type Integration = { export enum IntegrationLot { Push = 'PUSH', Sink = 'SINK', + Sse = 'SSE', Yank = 'YANK' } From 0e1c95701238c2d9e66d73532f680d8a26df44c0 Mon Sep 17 00:00:00 2001 From: Jacob-Tate Date: Mon, 5 Aug 2024 09:39:21 +0530 Subject: [PATCH 03/61] Finalizing implementation Cleaned up where some variables lived and documented important code. Removed unused imports Templatized duplicated code. Switched to an unbounded channel to allow users to make any length of background task polling periods. --- apps/backend/src/integrations.rs | 2 - apps/backend/src/integrations/komga.rs | 207 ++++++++++++++++++------- apps/backend/src/miscellaneous.rs | 71 ++------- apps/backend/src/models.rs | 11 -- 4 files changed, 167 insertions(+), 124 deletions(-) diff --git a/apps/backend/src/integrations.rs b/apps/backend/src/integrations.rs index a9b9d0c1b6..0f8fa40376 100644 --- a/apps/backend/src/integrations.rs +++ b/apps/backend/src/integrations.rs @@ -1,7 +1,6 @@ mod komga; use std::future::Future; - use anyhow::{anyhow, bail, Result}; use async_graphql::Result as GqlResult; use database::{MediaLot, MediaSource}; @@ -26,7 +25,6 @@ use sonarr_api_rs::{ }, models::{AddSeriesOptions as SonarrAddSeriesOptions, SeriesResource as SonarrSeriesResource}, }; - use crate::{ entities::{metadata, prelude::Metadata}, models::{audiobookshelf_models, media::CommitMediaInput}, diff --git a/apps/backend/src/integrations/komga.rs b/apps/backend/src/integrations/komga.rs index 0ef6fba410..0c5d0e65ef 100644 --- a/apps/backend/src/integrations/komga.rs +++ b/apps/backend/src/integrations/komga.rs @@ -1,18 +1,19 @@ +use std::collections::{hash_map::Entry, HashMap}; +use std::sync::{Mutex, OnceLock}; use anyhow::{anyhow, Result, Context}; use futures::StreamExt; use rust_decimal::Decimal; use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, ColumnTrait}; use sea_query::Expr; -use tokio::sync::{mpsc, mpsc::error::TryRecvError}; +use tokio::sync::{mpsc, mpsc::error::TryRecvError, mpsc::UnboundedReceiver}; use database::{MediaLot, MediaSource, }; use eventsource_stream::Eventsource; use rust_decimal::prelude::{FromPrimitive, Zero}; +use serde::de::DeserializeOwned; use super::{IntegrationMediaCollection, IntegrationMediaSeen, IntegrationService}; use crate::{ - miscellaneous::SSEObjects, entities::{metadata, prelude::Metadata}, utils::{get_base_http_client, }, - models::komga_events }; mod komga_book { @@ -75,6 +76,17 @@ mod komga_series { } impl Metadata { + /// Provided with a url this will extract the ID number from it. For example the url + /// https://myanimelist.net/manga/116778 will extract 116778 + /// + /// Currently only works for myanimelist and anilist as mangaupdates doesn't store the ID + /// in the url + /// + /// # Arguments + /// + /// * `url`: The url to extact from + /// + /// returns: Option The ID number if the extraction is successful fn extract_id(&self, url: String) -> Option { if let Ok(parsed_url) = Url::parse(&url) { parsed_url.path_segments() @@ -85,6 +97,16 @@ mod komga_series { } } + /// Extracts the list of providers with a MediaSource,ID Tuple + /// + /// Currently only works for myanimelist and anilist as mangaupdates doesn't store the ID + /// in the url + /// + /// Requires that the metadata is stored with the label anilist or myanimelist other + /// spellings wont work + /// + /// returns: Vec<(Option,Option)> list of providers with a + /// MediaSource,ID Tuple pub fn find_providers(&self) -> Vec<(Option,Option)> { let mut provider_links = vec![]; for link in self.links.iter() { @@ -113,7 +135,6 @@ mod komga_series { pub struct Item { pub id: String, pub name: String, - // pub number: Decimal, pub books_count: Decimal, pub books_read_count: Option, pub books_unread_count: Decimal, @@ -127,9 +148,52 @@ mod komga_series { } } -impl IntegrationService { +mod komga_events { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize, Clone)] + #[serde(rename_all = "camelCase")] + pub struct Data { + pub book_id: String, + pub user_id: String, + } +} + +struct KomgaEventHandler { + task: OnceLock<()>, + receiver: Mutex>>, +} - async fn sse_listener(sender: mpsc::Sender, +impl KomgaEventHandler { + pub const fn new() -> Self { + Self { + task: OnceLock::new(), + receiver: Mutex::new(None) + } + } + + pub fn get_receiver(&self) -> &Mutex>> { + &self.receiver + } + + pub fn get_task(&self) -> &OnceLock<()> { + &self.task + } +} + +impl IntegrationService { + /// Generates the sse listener for komga. This is intended to be run from another thread + /// if you run this in the main thread it will lock it up + /// + /// # Arguments + /// + /// * `sender`: The unbounded sender, lifetime of this sender is the lifetime of this + /// function so the sender doesn't need global lifetime + /// * `base_url`: URL for komga + /// * `cookie`: The komga cookie with the remember-me included + /// + /// returns: Never Returns + async fn sse_listener(sender: mpsc::UnboundedSender, base_url: String, cookie: String,) -> anyhow::Result<(), Box> { let client = get_base_http_client(&format!("{}/sse/v1/", base_url), None); @@ -148,11 +212,12 @@ impl IntegrationService { let event = event.context("Failed to get next event")?; tracing::trace!(?event, "Received SSE event"); - //TODO: Handle the deleted + // We could also handle ReadProgressDeleted here but I don't + // think we want to handle any deletions like this if event.event == "ReadProgressChanged" { match serde_json::from_str::(&event.data) { Ok(read_progress) => { - if sender.send(read_progress).await.is_err() { + if sender.send(read_progress).is_err() { tracing::debug!("Receiver dropped, exiting SSE listener"); break; } @@ -168,39 +233,52 @@ impl IntegrationService { } tracing::trace!("SSE listener finished"); - tokio::time::sleep(tokio::time::Duration::from_secs(5)).await; + tokio::time::sleep(tokio::time::Duration::from_secs(30)).await; } } - async fn fetch_book(client: &reqwest::Client, - book_id: &str, - cookie: &str) -> Result { - client - .get(format!("books/{}", book_id)) - .header("Cookie", cookie) - .send() - .await? - .error_for_status()? - .json::() - .await - .map_err(|e| anyhow!("Failed to parse book JSON: {}", e)) - } - - async fn fetch_series(client: &reqwest::Client, - series_id: &str, - cookie: &str) -> Result { + /// Fetches an API request to the provided client URL like + /// `https://acme.com/api_endpoint/api_id` + /// + /// # Arguments + /// + /// * `client`: Prepopulated client please use `get_base_http_client` to construct this + /// * `cookie`: The komga cookie with the remember-me included + /// * `api_endpoint`: Endpoint which comes after the base_url doesn't require a prepended `/` + /// * `api_id`: The ID of the object you are searching for added to the end of the + /// api_endpoint doesn't require a prepended `/` + /// + /// returns: Result This only preforms basic error handling on the json parsing + async fn fetch_api(client: &reqwest::Client, + cookie: &str, + api_endpoint: &str, + api_id: &str) -> Result { client - .get(format!("series/{}", series_id)) + .get(format!("{}/{}", api_endpoint, api_id)) .header("Cookie", cookie) .send() .await? .error_for_status()? - .json::() + .json::() .await - .map_err(|e| anyhow!("Failed to parse series JSON: {}", e)) + .map_err(|e| anyhow!("Failed to parse JSON: {}", e)) } + /// Finds the metadata provider and ID of the provided series + /// + /// # Arguments + /// + /// * `series`: The series object from which we want to grab the provider from. There + /// should be a links section which is populated with urls from which we + /// can extract the series ID. If not a simple search of the db for a manga with + /// the same title will be preformed + /// * `provider`: The preferred provider if this isn't available another will be used + /// in its place + /// * `db`: The metadata db connection + /// + /// returns: Result<(Option, Option), Error> This contains the mediasource + /// and the ID of the series. async fn find_provider_and_id( series: &komga_series::Item, provider: MediaSource, @@ -229,15 +307,27 @@ impl IntegrationService { fn calculate_percentage(curr_page: i32, total_page: i32) -> Decimal { if total_page == 0 { - return Decimal::zero(); // Handle division by zero case + return Decimal::zero(); } - // Perform the calculation using floating-point arithmetic let percentage = (curr_page as f64 / total_page as f64) * 100.0; - // Convert the result to Decimal Decimal::from_f64(percentage).unwrap_or(Decimal::zero()) } + + + /// Processes the events which are provided by the receiver + /// + /// # Arguments + /// + /// * `base_url`: URL for komga + /// * `cookie`: The komga cookie with the remember-me included + /// * `provider`: The preferred provider if this isn't available another will be used + /// in its place + /// * `data`: The data from the event + /// + /// returns: Option If the event had no issues processing contains the + /// media which was read otherwise none async fn process_events ( &self, base_url: &str, @@ -247,17 +337,14 @@ impl IntegrationService { ) -> Option { let client = get_base_http_client(&format!("{}/api/v1/", base_url), None); - // Fetch book and series data - let book = - IntegrationService::fetch_book(&client, &data.book_id, cookie).await.ok()?; - let series = - IntegrationService::fetch_series(&client, &book.series_id, cookie).await.ok()?; + let book: komga_book::Item = + Self::fetch_api(&client, cookie, "books", &data.book_id).await.ok()?; + let series: komga_series::Item = + Self::fetch_api(&client, cookie, "series", &book.series_id).await.ok()?; - // Find provider and ID let (source, id) = - IntegrationService::find_provider_and_id(&series, provider, &self.db).await.ok()?; + Self::find_provider_and_id(&series, provider, &self.db).await.ok()?; - // If no ID is found, return None let Some(id) = id else { tracing::debug!("No MAL URL or database entry found for manga: {}", series.name); return None; @@ -268,8 +355,8 @@ impl IntegrationService { lot: MediaLot::Manga, source: source.unwrap(), manga_chapter_number: Some(book.number), - progress: IntegrationService::calculate_percentage(book.read_progress.page, - book.media.pages_count), + progress: Self::calculate_percentage(book.read_progress.page, + book.media.pages_count), provider_watched_on: Some("Komga".to_string()), ..Default::default() }) @@ -280,21 +367,27 @@ impl IntegrationService { base_url: &str, cookie: &str, provider: MediaSource, - sse_lists: &SSEObjects ) -> Result<(Vec, Vec)> { - let mutex_receiver = sse_lists.get_komga_receiver(); + // This object needs global lifetime so we can continue to use the receiver + // If we ever create more SSE Objects we may want to implement a higher level + // Controller or make a housekeeping function to make sure the background + // threads are running correctly and kill them when the app is killed + // (though rust should handle this) + static SSE_LISTS: KomgaEventHandler = KomgaEventHandler ::new(); + + let mutex_receiver = SSE_LISTS.get_receiver(); let mut receiver = { let mut guard = mutex_receiver.lock().unwrap(); guard.take() }; if receiver.is_none() { - let (tx, rx) = mpsc::channel::(1000); + let (tx, rx) = mpsc::unbounded_channel::(); receiver = Some(rx); let base_url = base_url.to_string(); let cookie = cookie.to_string(); - let mutex_task = sse_lists.get_komga_task(); + let mutex_task = SSE_LISTS.get_task(); mutex_task.get_or_init(|| { tokio::spawn(async move { @@ -306,17 +399,24 @@ impl IntegrationService { }); } - let mut media_items = vec![]; + // Use hashmap here so we dont dupe pulls for a single book + let mut unique_media_items: HashMap = HashMap::new(); if let Some(mut recv) = receiver { loop { match recv.try_recv() { Ok(event) => { tracing::debug!("Received event {:?}", event); - match self.process_events(base_url, cookie, provider, event).await { - Some(processed_event) => - media_items.push(processed_event), - None => tracing::warn!("Failed to process event"), + match unique_media_items.entry(event.book_id.clone()) { + Entry::Vacant(entry) => { + if let Some(processed_event) = + self.process_events(base_url, cookie, provider, event.clone()).await { + entry.insert(processed_event); + } else { + tracing::warn!("Failed to process event for book_id: {}", event.book_id); + } + }, + _ => continue } }, Err(TryRecvError::Empty) => break, @@ -328,6 +428,9 @@ impl IntegrationService { mutex_receiver.lock().unwrap().replace(recv); } + let media_items = unique_media_items.into_values().collect(); + tracing::debug!("Media Items: {:?}", media_items); + Ok((media_items, vec![])) } -} \ No newline at end of file +} diff --git a/apps/backend/src/miscellaneous.rs b/apps/backend/src/miscellaneous.rs index f582c86fa1..61c396eba0 100644 --- a/apps/backend/src/miscellaneous.rs +++ b/apps/backend/src/miscellaneous.rs @@ -52,7 +52,6 @@ use sea_query::{ }; use serde::{Deserialize, Serialize}; use struson::writer::{JsonStreamWriter, JsonWriter}; -use tokio::sync::mpsc::Receiver; use crate::{ background::{ApplicationJob, CoreApplicationJob}, entities::{ @@ -95,7 +94,6 @@ use crate::{ CollectionToEntitySystemInformation, DefaultCollection, IdAndNamedObject, MediaStateChanged, SearchDetails, SearchInput, SearchResults, StoredUrl, StringIdObject, UserSummaryData, - komga_events }, providers::{ anilist::{ @@ -126,7 +124,6 @@ use crate::{ user_id_from_token, AUTHOR, SHOW_SPECIAL_SEASON_NAMES, TEMP_DIR, VERSION, }, }; -use std::sync::{Mutex, OnceLock}; type Provider = Box<(dyn MediaProvider + Send + Sync)>; @@ -1475,30 +1472,6 @@ impl MiscellaneousMutation { } } -pub struct SSEObjects { - komga_task: OnceLock<()>, - komga_receiver: Mutex>>, -} - -impl SSEObjects { - const fn new() -> SSEObjects { - SSEObjects { - komga_task: OnceLock::new(), - komga_receiver: Mutex::new(None) - } - } - - pub fn get_komga_receiver(&self) -> &Mutex>> { - &self.komga_receiver - } - - pub fn get_komga_task(&self) -> &OnceLock<()> { - &self.komga_task - } -} - -static SSE_LISTS: SSEObjects = SSEObjects::new(); - pub struct MiscellaneousService { pub db: DatabaseConnection, pub perform_application_job: MemoryStorage, @@ -2480,6 +2453,18 @@ impl MiscellaneousService { if progress == dec!(100) { last_seen.finished_on = ActiveValue::Set(Some(now.date_naive())); } + + // This is needed for manga as some of the apps will update in weird orders + // For example with komga mihon will update out of order to the server + if input.manga_chapter_number.is_some() { + last_seen.manga_extra_information = ActiveValue::set(Some( + SeenMangaExtraInformation { + chapter: input.manga_chapter_number, + volume: input.manga_volume_number, + } + )) + } + last_seen.update(&self.db).await.unwrap() } ProgressUpdateAction::ChangeState => { @@ -2546,37 +2531,6 @@ impl MiscellaneousService { None }; let manga_ei = if matches!(meta.lot, MediaLot::Manga) { - if respect_cache { - let test = SeenMangaExtraInformation { - volume: None, - chapter: input.manga_chapter_number - }; - - let all_seen = Seen::find() - .filter(seen::Column::Progress.eq(100)) - .filter(seen::Column::UserId.eq(user_id)) - .filter(seen::Column::State.ne(SeenState::Dropped)) - .filter(seen::Column::MetadataId.eq(&input.metadata_id)) - .filter( - seen::Column::MangaExtraInformation.eq(test) - // Expr::expr(Func::cast_as( - // Expr::col(seen::Column::MangaExtraInformation), - // Alias::new("text") - // )) - // .ilike(ilike_sql(format!("\"chapter\": {}", input.manga_chapter_number.unwrap()).to_string().as_ref())) - ) - .order_by_desc(seen::Column::LastUpdatedOn) - .all(&self.db) - .await - .unwrap(); - - if !all_seen.is_empty() { - return Ok(ProgressUpdateResultUnion::Error(ProgressUpdateError { - error: ProgressUpdateErrorVariant::AlreadySeen, - })); - } - } - Some(SeenMangaExtraInformation { chapter: input.manga_chapter_number, volume: input.manga_volume_number, @@ -5827,7 +5781,6 @@ impl MiscellaneousService { &specifics.komga_base_url.unwrap(), &specifics.komga_cookie.unwrap(), specifics.komga_provider.unwrap(), - &SSE_LISTS ) .await }, diff --git a/apps/backend/src/models.rs b/apps/backend/src/models.rs index 72abf980da..c52eb55341 100644 --- a/apps/backend/src/models.rs +++ b/apps/backend/src/models.rs @@ -1906,17 +1906,6 @@ pub mod importer { pub failed_items: Vec, } } - -pub mod komga_events { - use super::*; - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Data { - pub book_id: String, - pub user_id: String, - } -} pub mod audiobookshelf_models { use super::*; From 83105fef35a76c38fcadab36a9da5eeafd283b8d Mon Sep 17 00:00:00 2001 From: Jacob-Tate Date: Mon, 5 Aug 2024 10:00:33 +0530 Subject: [PATCH 04/61] Added Komga Documentation --- docs/content/integrations.md | 57 ++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/docs/content/integrations.md b/docs/content/integrations.md index fa359da0c3..07aee315fa 100644 --- a/docs/content/integrations.md +++ b/docs/content/integrations.md @@ -7,6 +7,7 @@ services about changes. They can be of following types: interval. - _Push_: Ryot sends data to an external service at a periodic interval. - _Sink_: An external client publishes progress updates to the Ryot server. +- _SSE_: A server publishes event data for clients to use. Ryot subscribes to the event stream to capture updates. ## Yank integrations @@ -137,3 +138,59 @@ TMDb ID attached to their metadata. the zipped addon to your Kodi instance. Once installed, it will be visible under the "Services" sub category named "Ryot". 4. Click on "Configure" to fill in the correct details. + + +## SSE integrations +For each integration you want to enable, credentials for the external server must be saved +to your profile. To do so, go to the "Settings" tab and add a new integration under the +"Integrations" tab. + +You can configure the interval at which the data is handled from the sse listener using the +`integration.sync_every_minutes` configuration key. Defaults to `5` (minutes). + +### Komga + +!!! warning + + This will only import media that are in progress. An import process will be added in + the future + +The [Komga](https://komga.org/) integration can sync all media if they +have a valid metadata provider. + +#### Komga side steps +If you use [Komf](https://github.com/Snd-R/komf) or some similar metadata provider these urls will be +populated automatically. If you don't use komf youll either need to manually add the manga to your collection +or you can perform the following steps. +1. Navigate to the manga +2. Open the edit tab +3. Navigate to the Links tab +4. Create a link named `AniList` or `MyAnimeList` providing the respective url (not case-sensitive) + +To retrieve your Cookie youll need to perform the following steps: +1. Log out of Komga +2. Log back in while selecting `Remember me` +3. Press F12 +4. Navigate to the Network tab +5. Refresh the page +6. Select any of the urls and look for the `Cookie` Header +7. Copy the entire cookie it should look something like this `remember-me=REDACTED; SESSION=REDACTED` + +If you are using NGINX as your reverse proxy youll probably need to add the following to your configuration +to allow the SSE stream to work correctly. + +```nginx +# Needs to be added to the location / section of the file +# If you dont the transfers will be chunked so youll get updates every 1000-2000 events +proxy_set_header Connection ''; +proxy_http_version 1.1; +chunked_transfer_encoding off; +proxy_buffering off; +proxy_cache off; +``` +#### Ryot side steps +1. Obtain your cookie as described above +2. Create the integration and select Komga as the source +3. Provide your BaseURL. Should look something like this `komga.acme.com` or `127.0.0.1:25600` +4. Provide your Cookie. +5. Provide your prefered metadata provider it will attempt the others if the first doesn't work and will fallback to title search otherwise From 9fc4687b4a12f01f0557696ef72de0aa599612d0 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Wed, 7 Aug 2024 19:02:06 -0700 Subject: [PATCH 05/61] Placed Cargo.toml in Alphabetic Order --- apps/backend/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/backend/Cargo.toml b/apps/backend/Cargo.toml index 8c15a5176d..98fc54514b 100644 --- a/apps/backend/Cargo.toml +++ b/apps/backend/Cargo.toml @@ -35,6 +35,7 @@ dotenvy = "=0.15.7" dotenvy_macro = { workspace = true } educe = { version = "=0.6.0", features = ["Debug"], default-features = false } enum_meta = "=0.7.0" +eventsource-stream = "0.2.3" flate2 = "=1.0.30" futures = "=0.3.30" graphql_client = "=0.14.0" @@ -82,4 +83,3 @@ tracing = { workspace = true } tracing-subscriber = "=0.3.18" logs-wheel = "=0.3.1" uuid = { version = "=1.10.0", features = ["v4"], default-features = false } -eventsource-stream = "0.2.3" From 3e1f70aa94ef20c7bdd97dde691c50c55471cf64 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Wed, 7 Aug 2024 19:14:49 -0700 Subject: [PATCH 06/61] Changed Komga into a Yank Integration. --- apps/backend/src/background.rs | 6 - apps/backend/src/miscellaneous.rs | 105 ++---------------- .../_dashboard.settings.integrations.tsx | 6 +- docs/content/integrations.md | 104 ++++++++--------- 4 files changed, 61 insertions(+), 160 deletions(-) diff --git a/apps/backend/src/background.rs b/apps/backend/src/background.rs index 26bb7f3f6b..80960ccbc8 100644 --- a/apps/backend/src/background.rs +++ b/apps/backend/src/background.rs @@ -46,8 +46,6 @@ pub async fn sync_integrations_data( _information: ScheduledJob, misc_service: Data>, ) -> Result<(), Error> { - tracing::trace!("Getting data from sse integrations for all users"); - misc_service.sse_integrations_data().await.unwrap(); tracing::trace!("Getting data from yanked integrations for all users"); misc_service.yank_integrations_data().await.unwrap(); tracing::trace!("Sending data for push integrations for all users"); @@ -87,10 +85,6 @@ pub async fn perform_core_application_job( misc_service .yank_integrations_data_for_user(&user_id) .await - .ok(); - misc_service - .sse_integrations_data_for_user(&user_id) - .await .is_ok() } CoreApplicationJob::BulkProgressUpdate(user_id, input) => misc_service diff --git a/apps/backend/src/miscellaneous.rs b/apps/backend/src/miscellaneous.rs index 61c396eba0..4364d544d8 100644 --- a/apps/backend/src/miscellaneous.rs +++ b/apps/backend/src/miscellaneous.rs @@ -5499,7 +5499,7 @@ impl MiscellaneousService { } let lot = match input.provider { IntegrationProvider::Audiobookshelf => IntegrationLot::Yank, - IntegrationProvider::Komga => IntegrationLot::SSE, + IntegrationProvider::Komga => IntegrationLot::Yank, IntegrationProvider::Radarr | IntegrationProvider::Sonarr => IntegrationLot::Push, _ => IntegrationLot::Sink, }; @@ -5754,83 +5754,6 @@ impl MiscellaneousService { .collect() } - pub async fn sse_integrations_data_for_user(&self, user_id: &String) -> Result { - let preferences = self.user_preferences(user_id).await?; - if preferences.general.disable_integrations { - return Ok(false); - } - let integrations = Integration::find() - .filter(integration::Column::UserId.eq(user_id)) - .all(&self.db) - .await?; - let mut progress_updates = vec![]; - let mut collection_updates = vec![]; - let mut to_update_integrations = vec![]; - let integration_service = self.get_integration_service(); - for integration in integrations.into_iter() { - if integration.is_disabled.unwrap_or_default() { - tracing::debug!("Integration {} is disabled", integration.id); - continue; - } - let response = match integration.provider { - IntegrationProvider::Komga => { - let specifics = integration.clone().provider_specifics.unwrap(); - - integration_service - .komga_progress( - &specifics.komga_base_url.unwrap(), - &specifics.komga_cookie.unwrap(), - specifics.komga_provider.unwrap(), - ) - .await - }, - _ => continue, - }; - if let Ok((seen_progress, collection_progress)) = response { - collection_updates.extend(collection_progress); - to_update_integrations.push(integration.id.clone()); - progress_updates.push((integration, seen_progress)); - } - } - for (integration, progress_updates) in progress_updates.into_iter() { - for pu in progress_updates.into_iter() { - self.integration_progress_update(&integration, pu, user_id) - .await - .trace_ok(); - } - } - for col_update in collection_updates.into_iter() { - let metadata::Model { id, .. } = self - .commit_metadata(CommitMediaInput { - lot: col_update.lot, - source: col_update.source, - identifier: col_update.identifier.clone(), - force_update: None, - }) - .await?; - self.add_entity_to_collection( - user_id, - ChangeCollectionToEntityInput { - creator_user_id: user_id.to_owned(), - collection_name: col_update.collection, - metadata_id: Some(id.clone()), - ..Default::default() - }, - ) - .await - .trace_ok(); - } - Integration::update_many() - .filter(integration::Column::Id.is_in(to_update_integrations)) - .col_expr( - integration::Column::LastTriggeredOn, - Expr::value(Utc::now()), - ) - .exec(&self.db) - .await?; - Ok(true) - } - pub async fn yank_integrations_data_for_user(&self, user_id: &String) -> Result { let preferences = self.user_preferences(user_id).await?; if preferences.general.disable_integrations { @@ -5861,6 +5784,17 @@ impl MiscellaneousService { ) .await }, + IntegrationProvider::Komga => { + let specifics = integration.clone().provider_specifics.unwrap(); + + integration_service + .komga_progress( + &specifics.komga_base_url.unwrap(), + &specifics.komga_cookie.unwrap(), + specifics.komga_provider.unwrap(), + ) + .await + }, _ => continue, }; if let Ok((seen_progress, collection_progress)) = response { @@ -5923,21 +5857,6 @@ impl MiscellaneousService { Ok(()) } - pub async fn sse_integrations_data(&self) -> Result<()> { - let users_with_integrations = Integration::find() - .filter(integration::Column::Lot.eq(IntegrationLot::SSE)) - .select_only() - .column(integration::Column::UserId) - .into_tuple::() - .all(&self.db) - .await?; - for user_id in users_with_integrations { - tracing::debug!("sse integrations data for user {}", user_id); - self.sse_integrations_data_for_user(&user_id).await?; - } - Ok(()) - } - pub async fn send_data_for_push_integrations(&self) -> Result<()> { let users_with_integrations = Integration::find() .filter(integration::Column::Lot.eq(IntegrationLot::Push)) diff --git a/apps/frontend/app/routes/_dashboard.settings.integrations.tsx b/apps/frontend/app/routes/_dashboard.settings.integrations.tsx index 0717164021..6763387c31 100644 --- a/apps/frontend/app/routes/_dashboard.settings.integrations.tsx +++ b/apps/frontend/app/routes/_dashboard.settings.integrations.tsx @@ -54,17 +54,15 @@ import { dayjsLib } from "~/lib/generals"; import { useConfirmSubmit, useUserCollections } from "~/lib/hooks"; import { createToastHeaders, serverGqlService } from "~/lib/utilities.server"; -const SSE_INTEGRATION = [ - IntegrationProvider.Komga, -] const YANK_INTEGRATIONS = [ IntegrationProvider.Audiobookshelf, + IntegrationProvider.Komga, ]; const PUSH_INTEGRATIONS = [ IntegrationProvider.Radarr, IntegrationProvider.Sonarr, ]; -const NO_SHOW_URL = [...SSE_INTEGRATION, ...YANK_INTEGRATIONS, ...PUSH_INTEGRATIONS]; +const NO_SHOW_URL = [...YANK_INTEGRATIONS, ...PUSH_INTEGRATIONS]; export const loader = unstable_defineLoader(async ({ request }) => { const [{ userIntegrations }] = await Promise.all([ diff --git a/docs/content/integrations.md b/docs/content/integrations.md index 07aee315fa..f9e8238d9e 100644 --- a/docs/content/integrations.md +++ b/docs/content/integrations.md @@ -7,7 +7,6 @@ services about changes. They can be of following types: interval. - _Push_: Ryot sends data to an external service at a periodic interval. - _Sink_: An external client publishes progress updates to the Ryot server. -- _SSE_: A server publishes event data for clients to use. Ryot subscribes to the event stream to capture updates. ## Yank integrations @@ -33,6 +32,53 @@ have a valid provider ID (Audible, ITunes or ISBN). 2. Go to your Ryot user settings and add the correct details as described in the [yank](#yank-integrations) section. +### Komga + +!!! warning + + This will only import media that are in progress. An import process will be added in + the future + +The [Komga](https://komga.org/) integration can sync all media if they +have a valid metadata provider. + +#### Komga side steps +If you use [Komf](https://github.com/Snd-R/komf) or some similar metadata provider these urls will be +populated automatically. If you don't use komf youll either need to manually add the manga to your collection +or you can perform the following steps. +1. Navigate to the manga +2. Open the edit tab +3. Navigate to the Links tab +4. Create a link named `AniList` or `MyAnimeList` providing the respective url (not case-sensitive) + +To retrieve your Cookie youll need to perform the following steps: +1. Log out of Komga +2. Log back in while selecting `Remember me` +3. Press F12 +4. Navigate to the Network tab +5. Refresh the page +6. Select any of the urls and look for the `Cookie` Header +7. Copy the entire cookie it should look something like this `remember-me=REDACTED; SESSION=REDACTED` + +If you are using NGINX as your reverse proxy youll probably need to add the following to your configuration +to allow the SSE stream to work correctly. + +```nginx +# Needs to be added to the location / section of the file +# If you dont the transfers will be chunked so youll get updates every 1000-2000 events +proxy_set_header Connection ''; +proxy_http_version 1.1; +chunked_transfer_encoding off; +proxy_buffering off; +proxy_cache off; +``` +#### Ryot side steps +1. Obtain your cookie as described above +2. Create the integration and select Komga as the source +3. Provide your BaseURL. Should look something like this `http://komga.acme.com` or `http://127.0.0.1:25600` +4. Provide your Cookie. +5. Provide your prefered metadata provider it will attempt the others if the first doesn't work and will fallback to title search otherwise + ## Push integrations Follow the same instructions as the [yank](#yank-integrations) integrations to add one. @@ -138,59 +184,3 @@ TMDb ID attached to their metadata. the zipped addon to your Kodi instance. Once installed, it will be visible under the "Services" sub category named "Ryot". 4. Click on "Configure" to fill in the correct details. - - -## SSE integrations -For each integration you want to enable, credentials for the external server must be saved -to your profile. To do so, go to the "Settings" tab and add a new integration under the -"Integrations" tab. - -You can configure the interval at which the data is handled from the sse listener using the -`integration.sync_every_minutes` configuration key. Defaults to `5` (minutes). - -### Komga - -!!! warning - - This will only import media that are in progress. An import process will be added in - the future - -The [Komga](https://komga.org/) integration can sync all media if they -have a valid metadata provider. - -#### Komga side steps -If you use [Komf](https://github.com/Snd-R/komf) or some similar metadata provider these urls will be -populated automatically. If you don't use komf youll either need to manually add the manga to your collection -or you can perform the following steps. -1. Navigate to the manga -2. Open the edit tab -3. Navigate to the Links tab -4. Create a link named `AniList` or `MyAnimeList` providing the respective url (not case-sensitive) - -To retrieve your Cookie youll need to perform the following steps: -1. Log out of Komga -2. Log back in while selecting `Remember me` -3. Press F12 -4. Navigate to the Network tab -5. Refresh the page -6. Select any of the urls and look for the `Cookie` Header -7. Copy the entire cookie it should look something like this `remember-me=REDACTED; SESSION=REDACTED` - -If you are using NGINX as your reverse proxy youll probably need to add the following to your configuration -to allow the SSE stream to work correctly. - -```nginx -# Needs to be added to the location / section of the file -# If you dont the transfers will be chunked so youll get updates every 1000-2000 events -proxy_set_header Connection ''; -proxy_http_version 1.1; -chunked_transfer_encoding off; -proxy_buffering off; -proxy_cache off; -``` -#### Ryot side steps -1. Obtain your cookie as described above -2. Create the integration and select Komga as the source -3. Provide your BaseURL. Should look something like this `komga.acme.com` or `127.0.0.1:25600` -4. Provide your Cookie. -5. Provide your prefered metadata provider it will attempt the others if the first doesn't work and will fallback to title search otherwise From 86e2c5c735970893912d04a3d1f7b665bb789d1e Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Wed, 7 Aug 2024 19:39:51 -0700 Subject: [PATCH 07/61] Removed provider specifics as it wasnt really needed. --- apps/backend/src/entities/integration.rs | 1 + apps/backend/src/miscellaneous.rs | 7 ------ .../_dashboard.settings.integrations.tsx | 25 +------------------ libs/database/src/definitions.rs | 1 - libs/generated/src/graphql/backend/gql.ts | 4 +-- libs/generated/src/graphql/backend/graphql.ts | 25 ++----------------- libs/graphql/src/backend/queries/combined.gql | 3 --- 7 files changed, 6 insertions(+), 60 deletions(-) diff --git a/apps/backend/src/entities/integration.rs b/apps/backend/src/entities/integration.rs index e949cddbd9..aec15371a2 100644 --- a/apps/backend/src/entities/integration.rs +++ b/apps/backend/src/entities/integration.rs @@ -26,6 +26,7 @@ pub struct Model { pub created_on: DateTimeUtc, #[graphql(skip_input)] pub last_triggered_on: Option, + #[graphql(skip)] #[sea_orm(column_type = "Json")] pub provider_specifics: Option, } diff --git a/apps/backend/src/miscellaneous.rs b/apps/backend/src/miscellaneous.rs index 4364d544d8..588d814226 100644 --- a/apps/backend/src/miscellaneous.rs +++ b/apps/backend/src/miscellaneous.rs @@ -5543,13 +5543,6 @@ impl MiscellaneousService { if let Some(d) = input.is_disabled { db_integration.is_disabled = ActiveValue::Set(Some(d)); } - if let Some(d) = input.provider_specifics { - let mut existing_specifics = - db_integration.provider_specifics.clone().unwrap().clone().unwrap().clone(); - - existing_specifics.komga_provider = d.komga_provider; - db_integration.provider_specifics = ActiveValue::Set(Some(existing_specifics)); - } db_integration.update(&self.db).await?; Ok(true) } diff --git a/apps/frontend/app/routes/_dashboard.settings.integrations.tsx b/apps/frontend/app/routes/_dashboard.settings.integrations.tsx index 6763387c31..cb99275904 100644 --- a/apps/frontend/app/routes/_dashboard.settings.integrations.tsx +++ b/apps/frontend/app/routes/_dashboard.settings.integrations.tsx @@ -189,11 +189,6 @@ const updateSchema = z.object({ minimumProgress: z.string().optional(), maximumProgress: z.string().optional(), isDisabled: zx.CheckboxAsString.optional(), - providerSpecifics: z - .object({ - komgaProvider: z.nativeEnum(MediaSource), - }) - .optional(), }); export default function Page() { @@ -471,7 +466,7 @@ const CreateIntegrationModal = (props: { label="Select a provider" name="providerSpecifics.komgaProvider" required - data={[MediaSource.Anilist,MediaSource.Mal].map((is) => ({ + data={[MediaSource.Anilist, MediaSource.Mal].map((is) => ({ label: changeCase(is), value: is, }))} @@ -585,24 +580,6 @@ const UpdateIntegrationModal = (props: { /> ) : null} - {match(props.updateIntegrationData.provider) - .with(IntegrationProvider.Komga, () => ( - <> - ({ + label: changeCase(is), + value: is, + }))} + /> + + )) .with(IntegrationProvider.Plex, () => ( <> , pub audiobookshelf_base_url: Option, pub audiobookshelf_token: Option, + pub komga_base_url: Option, + pub komga_username: Option, + pub komga_password: Option, + pub komga_provider: Option, pub radarr_base_url: Option, pub radarr_api_key: Option, pub radarr_profile_id: Option, diff --git a/crates/providers/src/manga_updates.rs b/crates/providers/src/manga_updates.rs index 437f499c17..009f15dc17 100644 --- a/crates/providers/src/manga_updates.rs +++ b/crates/providers/src/manga_updates.rs @@ -141,7 +141,12 @@ struct MetadataSearchResponse { } impl MangaUpdatesService { - fn extract_status(&self, input: &str) -> (Option, Option) { + fn extract_status(&self, input: Option) -> (Option, Option) { + if input.is_none() { + return (None, None); + } + + let input = input.unwrap(); let first_part = input.split("
").next().unwrap_or("").trim(); let parts: Vec<&str> = first_part.split_whitespace().collect(); @@ -324,7 +329,7 @@ impl MediaProvider for MangaUpdatesService { } } - let (volumes, status) = self.extract_status(&data.status.clone().unwrap()); + let (volumes, status) = self.extract_status(data.status.clone()); Ok(MediaDetails { identifier: data.series_id.unwrap().to_string(), diff --git a/crates/services/integration/Cargo.toml b/crates/services/integration/Cargo.toml index d102cc4105..faf60b36e0 100644 --- a/crates/services/integration/Cargo.toml +++ b/crates/services/integration/Cargo.toml @@ -8,9 +8,11 @@ anyhow = { workspace = true } application-utils = { path = "../../utils/application" } async-graphql = { workspace = true } enums = { path = "../../enums" } +eventsource-stream = "=0.2.3" database-models = { path = "../../models/database" } database-utils = { path = "../../utils/database" } media-models = { path = "../../models/media" } +openidconnect = { workspace = true } providers = { path = "../../providers" } radarr-api-rs = "=3.0.1" regex = { workspace = true } @@ -23,5 +25,6 @@ serde = { workspace = true } serde_json = { workspace = true } sonarr-api-rs = "=3.0.0" specific-models = { path = "../../models/specific" } +tokio = "1.39.2" tracing = { workspace = true } traits = { path = "../../traits" } diff --git a/crates/services/integration/src/komga.rs b/crates/services/integration/src/komga.rs new file mode 100644 index 0000000000..ff2e40cb2d --- /dev/null +++ b/crates/services/integration/src/komga.rs @@ -0,0 +1,466 @@ +use std::{ + collections::{hash_map::Entry, HashMap}, + sync::{Mutex, OnceLock}, +}; + +use anyhow::{anyhow, Context, Result}; +use async_graphql::futures_util::StreamExt; +use eventsource_stream::Eventsource; +use rust_decimal::{ + Decimal, + prelude::{FromPrimitive, Zero}, +}; +use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; +use sea_query::Expr; +use serde::de::DeserializeOwned; +use tokio::sync::{mpsc, mpsc::error::TryRecvError, mpsc::UnboundedReceiver}; + +use database_models::{ + metadata, + prelude::Metadata +}; + +use crate::{ + get_base_http_client, + MediaLot, + MediaSource, +}; + +use super::{IntegrationMediaCollection, IntegrationMediaSeen, IntegrationService}; + +mod komga_book { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Link { + pub label: String, + pub url: String, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Media { + pub pages_count: i32, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Metadata { + pub number: String, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct ReadProgress { + pub page: i32, + pub completed: bool, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Item { + pub id: String, + pub name: String, + pub series_id: String, + pub media: Media, + pub number: i32, + pub metadata: Metadata, + pub read_progress: ReadProgress, + } +} + +mod komga_series { + use openidconnect::url::Url; + use serde::{Deserialize, Serialize}; + + use crate::MediaSource; + + use super::*; + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Link { + pub label: String, + pub url: String, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Metadata { + pub links: Vec, + } + + impl Metadata { + /// Provided with a url this will extract the ID number from it. For example the url + /// https://myanimelist.net/manga/116778 will extract 116778 + /// + /// Currently only works for myanimelist and anilist as mangaupdates doesn't store the ID + /// in the url + /// + /// # Arguments + /// + /// * `url`: The url to extact from + /// + /// returns: The ID number if the extraction is successful + fn extract_id(&self, url: String) -> Option { + if let Ok(parsed_url) = Url::parse(&url) { + parsed_url + .path_segments() + .and_then(|segments| segments.collect::>().get(1).cloned()) + .map(String::from) + } else { + None + } + } + + /// Extracts the list of providers with a MediaSource,ID Tuple + /// + /// Currently only works for myanimelist and anilist as mangaupdates doesn't store + /// the ID in the url + /// + /// Requires that the metadata is stored with the label anilist or myanimelist + /// other spellings wont work + /// + /// returns: list of providers with a MediaSource, ID Tuple + pub fn find_providers(&self) -> Vec<(MediaSource, Option)> { + let mut provider_links = vec![]; + for link in self.links.iter() { + // NOTE: manga_updates doesn't work here because the ID isn't in the url + let source = match link.label.to_lowercase().as_str() { + "anilist" => MediaSource::Anilist, + "myanimelist" => MediaSource::Mal, + _ => continue, + }; + + let id = self.extract_id(link.url.clone()); + provider_links.push((source, id)); + } + + provider_links.sort_by_key(|a| a.1.clone()); + provider_links + } + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Item { + pub id: String, + pub name: String, + pub books_count: Decimal, + pub books_read_count: Option, + pub books_unread_count: Decimal, + pub metadata: Metadata, + } + + #[derive(Debug, Serialize, Deserialize)] + #[serde(rename_all = "camelCase")] + pub struct Response { + pub content: Vec, + } +} + +mod komga_events { + use serde::{Deserialize, Serialize}; + + #[derive(Debug, Serialize, Deserialize, Clone)] + #[serde(rename_all = "camelCase")] + pub struct Data { + pub book_id: String, + pub user_id: String, + } +} + +struct KomgaEventHandler { + task: OnceLock<()>, + receiver: Mutex>>, +} + +impl KomgaEventHandler { + pub const fn new() -> Self { + Self { + task: OnceLock::new(), + receiver: Mutex::new(None), + } + } + + pub fn get_receiver(&self) -> &Mutex>> { + &self.receiver + } + + pub fn get_task(&self) -> &OnceLock<()> { + &self.task + } +} + +impl IntegrationService { + /// Generates the sse listener for komga. This is intended to be run from another + /// thread if you run this in the main thread it will lock it up + /// + /// # Arguments + /// + /// * `sender`: The unbounded sender, lifetime of this sender is the lifetime of this + /// function so the sender doesn't need global lifetime + /// * `base_url`: URL for komga + /// * `cookie`: The komga cookie with the remember-me included + /// + /// returns: Never Returns + async fn sse_listener( + sender: mpsc::UnboundedSender, + base_url: String, + komga_username: String, + komga_password: String, + ) -> Result<(), Box> { + let client = get_base_http_client(&format!("{}/sse/v1/", base_url), None); + + loop { + let response = client + .get("events") + .basic_auth(komga_username.to_string(), Some(komga_password.to_string())) + .send() + .await + .context("Failed to send request")?; + + let mut stream = response.bytes_stream().eventsource(); + + while let Some(event) = stream.next().await { + let event = event.context("Failed to get next event")?; + tracing::trace!(?event, "Received SSE event"); + + // We could also handle ReadProgressDeleted here but I don't + // think we want to handle any deletions like this + if event.event == "ReadProgressChanged" { + match serde_json::from_str::(&event.data) { + Ok(read_progress) => { + if sender.send(read_progress).is_err() { + tracing::debug!("Receiver dropped, exiting SSE listener"); + break; + } + } + Err(e) => { + tracing::warn!(error = ?e, data = ?event.data, + "Failed to parse ReadProgressChanged event data"); + } + } + } else { + tracing::trace!(event_type = ?event.event, "Received unhandled event type"); + } + } + + tracing::trace!("SSE listener finished"); + tokio::time::sleep(tokio::time::Duration::from_secs(30)).await; + } + } + + /// Fetches an API request to the provided client URL like + /// `https://acme.com/api_endpoint/api_id` + /// + /// # Arguments + /// + /// * `client`: Prepopulated client please use `get_base_http_client` to construct this + /// * `cookie`: The komga cookie with the remember-me included + /// * `api_endpoint`: Endpoint which comes after the base_url doesn't require a + /// prepended `/` + /// * `api_id`: The ID of the object you are searching for added to the end of the + /// api_endpoint doesn't require a prepended `/` + /// + /// returns: This only preforms basic error handling on the json parsing + async fn fetch_api( + client: &reqwest::Client, + komga_username: &str, + komga_password: &str, + api_endpoint: &str, + api_id: &str, + ) -> Result { + client + .get(format!("{}/{}", api_endpoint, api_id)) + .basic_auth(komga_username, Some(komga_password)) + .send() + .await? + .error_for_status()? + .json::() + .await + .map_err(|e| anyhow!("Failed to parse JSON: {}", e)) + } + + /// Finds the metadata provider and ID of the provided series + /// + /// # Arguments + /// + /// * `series`: The series object from which we want to grab the provider from. There + /// should be a links section which is populated with urls from which we + /// can extract the series ID. If not a simple search of the db for a manga + /// with the same title will be preformed + /// * `provider`: The preferred provider if this isn't available another will be used + /// in its place + /// * `db`: The metadata db connection + /// + /// returns: This contains the MediaSource and the ID of the series. + async fn find_provider_and_id( + series: &komga_series::Item, + provider: MediaSource, + db: &DatabaseConnection, + ) -> Result<(MediaSource, Option)> { + let providers = series.metadata.find_providers(); + if !providers.is_empty() { + Ok(providers + .iter() + .find(|x| x.0 == provider) + .cloned() + .or_else(|| providers.first().cloned()) + .unwrap_or_default()) + } else { + let db_manga = Metadata::find() + .filter(metadata::Column::Lot.eq(MediaLot::Manga)) + .filter(Expr::col(metadata::Column::Title).eq(&series.name)) + .one(db) + .await?; + + Ok(db_manga + .map(|manga| (manga.source, Some(manga.identifier))) + .unwrap_or_default()) + } + } + + fn calculate_percentage(curr_page: i32, total_page: i32) -> Decimal { + if total_page == 0 { + return Decimal::zero(); + } + + let percentage = (curr_page as f64 / total_page as f64) * 100.0; + + Decimal::from_f64(percentage).unwrap_or(Decimal::zero()) + } + + /// Processes the events which are provided by the receiver + /// + /// # Arguments + /// + /// * `base_url`: URL for komga + /// * `cookie`: The komga cookie with the remember-me included + /// * `provider`: The preferred provider if this isn't available another will be used + /// in its place + /// * `data`: The data from the event + /// + /// returns: If the event had no issues processing contains the media which was read + /// otherwise none + async fn process_events( + &self, + base_url: &str, + komga_username: &str, + komga_password: &str, + provider: MediaSource, + data: komga_events::Data, + ) -> Option { + let client = get_base_http_client(&format!("{}/api/v1/", base_url), None); + + let book: komga_book::Item = Self::fetch_api(&client, komga_username, komga_password, "books", &data.book_id) + .await + .ok()?; + let series: komga_series::Item = + Self::fetch_api(&client, komga_username, komga_password, "series", &book.series_id) + .await + .ok()?; + + let (source, id) = Self::find_provider_and_id(&series, provider, &self.db) + .await + .ok()?; + + let Some(id) = id else { + tracing::debug!( + "No MAL URL or database entry found for manga: {}", + series.name + ); + return None; + }; + + Some(IntegrationMediaSeen { + identifier: id, + lot: MediaLot::Manga, + source, + manga_chapter_number: Some(book.metadata.number.parse().unwrap_or_default()), + progress: Self::calculate_percentage(book.read_progress.page, book.media.pages_count), + provider_watched_on: Some("Komga".to_string()), + ..Default::default() + }) + } + + pub async fn komga_progress( + &self, + base_url: &str, + komga_username: &str, + komga_password: &str, + provider: MediaSource, + ) -> Result<(Vec, Vec)> { + // DEV: This object needs global lifetime so we can continue to use the receiver If + // we ever create more SSE Objects we may want to implement a higher level + // Controller or make a housekeeping function to make sure the background threads + // are running correctly and kill them when the app is killed (though rust should + // handle this). + static SSE_LISTS: KomgaEventHandler = KomgaEventHandler::new(); + + let mutex_receiver = SSE_LISTS.get_receiver(); + let mut receiver = { + let mut guard = mutex_receiver.lock().unwrap(); + guard.take() + }; + + if receiver.is_none() { + let (tx, rx) = mpsc::unbounded_channel::(); + receiver = Some(rx); + + let base_url = base_url.to_string(); + let mutex_task = SSE_LISTS.get_task(); + let komga_username = komga_username.to_string(); + let komga_password = komga_password.to_string(); + + mutex_task.get_or_init(|| { + tokio::spawn(async move { + if let Err(e) = IntegrationService::sse_listener(tx, base_url, komga_username, komga_password).await { + tracing::error!("SSE listener error: {}", e); + } + }); + }); + } + + // Use hashmap here so we don't dupe pulls for a single book + let mut unique_media_items: HashMap = HashMap::new(); + + if let Some(mut recv) = receiver { + loop { + match recv.try_recv() { + Ok(event) => { + tracing::debug!("Received event {:?}", event); + match unique_media_items.entry(event.book_id.clone()) { + Entry::Vacant(entry) => { + if let Some(processed_event) = self + .process_events(base_url, komga_username, komga_password, provider, event.clone()) + .await + { + entry.insert(processed_event); + } else { + tracing::warn!( + "Failed to process event for book_id: {}", + event.book_id + ); + } + } + _ => continue, + } + } + Err(TryRecvError::Empty) => break, + Err(e) => return Err(anyhow::anyhow!("Receiver error: {}", e)), + } + } + + // Put the receiver back + mutex_receiver.lock().unwrap().replace(recv); + } + + let mut media_items: Vec = unique_media_items.into_values().collect(); + media_items.sort_by(|a, b| a.manga_chapter_number.cmp(&b.manga_chapter_number)); + tracing::debug!("Media Items: {:?}", media_items); + + Ok((media_items, vec![])) + } +} diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index ac82d382c1..30003dc49b 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -32,6 +32,8 @@ use sonarr_api_rs::{ use specific_models::audiobookshelf as audiobookshelf_models; use traits::TraceOk; +mod komga; + #[derive(Debug)] pub struct IntegrationService { db: DatabaseConnection, diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index f953f87c44..d689cd721b 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -1159,6 +1159,17 @@ impl MiscellaneousService { if progress == dec!(100) { last_seen.finished_on = ActiveValue::Set(Some(now.date_naive())); } + + // This is needed for manga as some of the apps will update in weird orders + // For example with komga mihon will update out of order to the server + if input.manga_chapter_number.is_some() { + last_seen.manga_extra_information = + ActiveValue::set(Some(SeenMangaExtraInformation { + chapter: input.manga_chapter_number, + volume: input.manga_volume_number, + })) + } + last_seen.update(&self.db).await.unwrap() } ProgressUpdateAction::ChangeState => { @@ -4153,6 +4164,7 @@ impl MiscellaneousService { } let lot = match input.provider { IntegrationProvider::Audiobookshelf => IntegrationLot::Yank, + IntegrationProvider::Komga => IntegrationLot::Yank, IntegrationProvider::Radarr | IntegrationProvider::Sonarr => IntegrationLot::Push, _ => IntegrationLot::Sink, }; @@ -4430,6 +4442,17 @@ impl MiscellaneousService { ) .await } + IntegrationProvider::Komga => { + let specifics = integration.clone().provider_specifics.unwrap(); + integration_service + .komga_progress( + &specifics.komga_base_url.unwrap(), + &specifics.komga_username.unwrap(), + &specifics.komga_password.unwrap(), + specifics.komga_provider.unwrap(), + ) + .await + } _ => continue, }; if let Ok((seen_progress, collection_progress)) = response { @@ -4477,7 +4500,7 @@ impl MiscellaneousService { Ok(true) } - pub async fn yank_integrations_data(&self) -> Result<()> { + async fn yank_integrations_data(&self) -> Result<()> { let users_with_integrations = Integration::find() .filter(integration::Column::Lot.eq(IntegrationLot::Yank)) .select_only() @@ -4492,6 +4515,12 @@ impl MiscellaneousService { Ok(()) } + pub async fn sync_integrations_data(&self) -> Result<()> { + tracing::trace!("Syncing integrations data..."); + self.yank_integrations_data().await.unwrap(); + Ok(()) + } + pub async fn handle_entity_added_to_collection_event( &self, user_id: String, diff --git a/docs/content/integrations.md b/docs/content/integrations.md index 9ed08e18b2..964d3b7877 100644 --- a/docs/content/integrations.md +++ b/docs/content/integrations.md @@ -46,6 +46,30 @@ have a valid provider ID (Audible, ITunes or ISBN). 2. Go to your Ryot user settings and add the correct details as described in the [yank](#yank-integrations) section. +### Komga + +The [Komga](https://komga.org/) integration can sync all media if they +have a valid metadata provider. + +#### Steps + +If you use [Komf](https://github.com/Snd-R/komf) or some similar metadata provider these +urls will be populated automatically. If you don't use komf you will either need to +manually add the manga to your collection or you can perform the following steps. + +1. Navigate to the manga +2. Open the edit tab +3. Navigate to the Links tab +4. Create a link named `AniList` or `MyAnimeList` providing the respective url (not case-sensitive) + +Then perform these steps on Ryot + +1. Create the integration and select Komga as the source +2. Provide your BaseURL. Should look something like this `http://komga.acme.com` or `http://127.0.0.1:25600` +3. Provide your Username and Password. +4. Provide your preferred metadata provider. Ryot will attempt the others if the preferred + is unavailable and will fallback to title search otherwise. + ## Sink integrations These work via webhooks wherein an external service can inform Ryot about a change. All diff --git a/libs/generated/src/graphql/backend/graphql.ts b/libs/generated/src/graphql/backend/graphql.ts index dd50d61992..0794d6bca9 100644 --- a/libs/generated/src/graphql/backend/graphql.ts +++ b/libs/generated/src/graphql/backend/graphql.ts @@ -35,15 +35,23 @@ export type Scalars = { * * `2000-02-24` */ NaiveDate: { input: string; output: string; } + /** + * ISO 8601 combined date and time without timezone. + * + * # Examples + * + * * `2015-07-01T08:59:60.123`, + */ + NaiveDateTime: { input: any; output: any; } }; export type AnimeAiringScheduleSpecifics = { - airingAt: Scalars['DateTime']['output']; + airingAt: Scalars['NaiveDateTime']['output']; episode: Scalars['Int']['output']; }; export type AnimeAiringScheduleSpecificsInput = { - airingAt: Scalars['DateTime']['input']; + airingAt: Scalars['NaiveDateTime']['input']; episode: Scalars['Int']['input']; }; @@ -730,6 +738,7 @@ export enum IntegrationProvider { Emby = 'EMBY', Jellyfin = 'JELLYFIN', Kodi = 'KODI', + Komga = 'KOMGA', Plex = 'PLEX', Radarr = 'RADARR', Sonarr = 'SONARR' @@ -738,6 +747,10 @@ export enum IntegrationProvider { export type IntegrationSourceSpecificsInput = { audiobookshelfBaseUrl?: InputMaybe; audiobookshelfToken?: InputMaybe; + komgaBaseUrl?: InputMaybe; + komgaPassword?: InputMaybe; + komgaProvider?: InputMaybe; + komgaUsername?: InputMaybe; plexUsername?: InputMaybe; radarrApiKey?: InputMaybe; radarrBaseUrl?: InputMaybe; From 1b02fc541399e88ee73e680f24599d85d64aea58 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Fri, 16 Aug 2024 20:54:32 -0700 Subject: [PATCH 30/61] Resolve merge errors --- Cargo.lock | 75 ---- apps/backend/Cargo.toml | 8 - apps/backend/src/integrations/komga.rs | 453 ------------------------- apps/backend/src/integrations/mod.rs | 0 apps/backend/src/models.rs | 0 crates/enums/src/lib.rs | 1 - crates/services/integration/src/lib.rs | 37 -- 7 files changed, 574 deletions(-) delete mode 100644 apps/backend/src/integrations/komga.rs delete mode 100644 apps/backend/src/integrations/mod.rs delete mode 100644 apps/backend/src/models.rs diff --git a/Cargo.lock b/Cargo.lock index 89db106038..2b9ceab77d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5072,81 +5072,6 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" -[[package]] -name = "ryot" -version = "0.1.0" -dependencies = [ - "anyhow", - "apalis", - "argon2", - "askama", - "async-graphql", - "async-graphql-axum", - "async-trait", - "aws-sdk-s3", - "axum", - "boilermates", - "cached", - "chrono", - "chrono-tz", - "compile-time", - "config", - "const-str", - "convert_case", - "csv", - "data-encoding", - "database", - "derive_more 1.0.0-beta.6", - "dotenvy", - "dotenvy_macro", - "educe 0.6.0", - "enum_meta", - "eventsource-stream", - "flate2", - "futures", - "graphql_client", - "hashbag", - "http 1.1.0", - "isolang", - "itertools 0.13.0", - "jsonwebtoken", - "lettre", - "logs-wheel", - "markdown", - "mime_guess", - "nanoid", - "openidconnect", - "paginate", - "radarr-api-rs", - "rand 0.9.0-alpha.2", - "regex", - "reqwest 0.11.23", - "rs-utils", - "rust_decimal", - "rust_decimal_macros", - "rust_iso3166", - "schematic", - "scraper", - "sea-orm", - "sea-orm-migration", - "sea-query", - "serde", - "serde-xml-rs", - "serde_json", - "serde_with 3.9.0", - "slug", - "sonarr-api-rs", - "strum", - "struson", - "tokio", - "tokio-util", - "tower", - "tower-http", - "tracing", - "tracing-subscriber", - "uuid", -] - [[package]] name = "ryu" version = "1.0.18" diff --git a/apps/backend/Cargo.toml b/apps/backend/Cargo.toml index 5287388b56..c17ebcc8ce 100644 --- a/apps/backend/Cargo.toml +++ b/apps/backend/Cargo.toml @@ -16,14 +16,6 @@ chrono = { workspace = true } chrono-tz = { workspace = true } config = { path = "../../crates/config" } dotenvy = "=0.15.7" -dotenvy_macro = { workspace = true } -educe = { version = "=0.6.0", features = ["Debug"], default-features = false } -enum_meta = "=0.7.0" -eventsource-stream = "=0.2.3" -flate2 = "=1.0.30" -futures = "=0.3.30" -graphql_client = "=0.14.0" -hashbag = "=0.1.12" http = "=1.1.0" itertools = { workspace = true } logs-wheel = "=0.3.1" diff --git a/apps/backend/src/integrations/komga.rs b/apps/backend/src/integrations/komga.rs deleted file mode 100644 index c4ab251194..0000000000 --- a/apps/backend/src/integrations/komga.rs +++ /dev/null @@ -1,453 +0,0 @@ -use std::{ - collections::{hash_map::Entry, HashMap}, - sync::{Mutex, OnceLock}, -}; - -use anyhow::{anyhow, Context, Result}; -use database::{MediaLot, MediaSource}; -use eventsource_stream::Eventsource; -use futures::StreamExt; -use rust_decimal::{ - Decimal, - prelude::{FromPrimitive, Zero}, -}; -use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; -use sea_query::Expr; -use serde::de::DeserializeOwned; -use tokio::sync::{mpsc, mpsc::error::TryRecvError, mpsc::UnboundedReceiver}; - -use super::{IntegrationMediaCollection, IntegrationMediaSeen, IntegrationService}; - -use crate::{ - entities::{metadata, prelude::Metadata}, - utils::get_base_http_client, -}; - -mod komga_book { - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Link { - pub label: String, - pub url: String, - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Media { - pub pages_count: i32, - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Metadata { - pub number: String, - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct ReadProgress { - pub page: i32, - pub completed: bool, - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Item { - pub id: String, - pub name: String, - pub series_id: String, - pub media: Media, - pub number: i32, - pub metadata: Metadata, - pub read_progress: ReadProgress, - } -} - -mod komga_series { - use super::*; - use openidconnect::url::Url; - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Link { - pub label: String, - pub url: String, - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Metadata { - pub links: Vec, - } - - impl Metadata { - /// Provided with a url this will extract the ID number from it. For example the url - /// https://myanimelist.net/manga/116778 will extract 116778 - /// - /// Currently only works for myanimelist and anilist as mangaupdates doesn't store the ID - /// in the url - /// - /// # Arguments - /// - /// * `url`: The url to extact from - /// - /// returns: The ID number if the extraction is successful - fn extract_id(&self, url: String) -> Option { - if let Ok(parsed_url) = Url::parse(&url) { - parsed_url - .path_segments() - .and_then(|segments| segments.collect::>().get(1).cloned()) - .map(String::from) - } else { - None - } - } - - /// Extracts the list of providers with a MediaSource,ID Tuple - /// - /// Currently only works for myanimelist and anilist as mangaupdates doesn't store - /// the ID in the url - /// - /// Requires that the metadata is stored with the label anilist or myanimelist - /// other spellings wont work - /// - /// returns: list of providers with a MediaSource, ID Tuple - pub fn find_providers(&self) -> Vec<(MediaSource, Option)> { - let mut provider_links = vec![]; - for link in self.links.iter() { - // NOTE: manga_updates doesn't work here because the ID isn't in the url - let source = match link.label.to_lowercase().as_str() { - "anilist" => MediaSource::Anilist, - "myanimelist" => MediaSource::Mal, - _ => continue, - }; - - let id = self.extract_id(link.url.clone()); - provider_links.push((source, id)); - } - - provider_links.sort_by_key(|a| a.1.clone()); - provider_links - } - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Item { - pub id: String, - pub name: String, - pub books_count: Decimal, - pub books_read_count: Option, - pub books_unread_count: Decimal, - pub metadata: Metadata, - } - - #[derive(Debug, Serialize, Deserialize)] - #[serde(rename_all = "camelCase")] - pub struct Response { - pub content: Vec, - } -} - -mod komga_events { - use serde::{Deserialize, Serialize}; - - #[derive(Debug, Serialize, Deserialize, Clone)] - #[serde(rename_all = "camelCase")] - pub struct Data { - pub book_id: String, - pub user_id: String, - } -} - -struct KomgaEventHandler { - task: OnceLock<()>, - receiver: Mutex>>, -} - -impl KomgaEventHandler { - pub const fn new() -> Self { - Self { - task: OnceLock::new(), - receiver: Mutex::new(None), - } - } - - pub fn get_receiver(&self) -> &Mutex>> { - &self.receiver - } - - pub fn get_task(&self) -> &OnceLock<()> { - &self.task - } -} - -impl IntegrationService { - /// Generates the sse listener for komga. This is intended to be run from another - /// thread if you run this in the main thread it will lock it up - /// - /// # Arguments - /// - /// * `sender`: The unbounded sender, lifetime of this sender is the lifetime of this - /// function so the sender doesn't need global lifetime - /// * `base_url`: URL for komga - /// * `cookie`: The komga cookie with the remember-me included - /// - /// returns: Never Returns - async fn sse_listener( - sender: mpsc::UnboundedSender, - base_url: String, - cookie: String, - ) -> anyhow::Result<(), Box> { - let client = get_base_http_client(&format!("{}/sse/v1/", base_url), None); - - loop { - let response = client - .get("events") - .header("Cookie", cookie.clone()) - .send() - .await - .context("Failed to send request")?; - - let mut stream = response.bytes_stream().eventsource(); - - while let Some(event) = stream.next().await { - let event = event.context("Failed to get next event")?; - tracing::trace!(?event, "Received SSE event"); - - // We could also handle ReadProgressDeleted here but I don't - // think we want to handle any deletions like this - if event.event == "ReadProgressChanged" { - match serde_json::from_str::(&event.data) { - Ok(read_progress) => { - if sender.send(read_progress).is_err() { - tracing::debug!("Receiver dropped, exiting SSE listener"); - break; - } - } - Err(e) => { - tracing::warn!(error = ?e, data = ?event.data, - "Failed to parse ReadProgressChanged event data"); - } - } - } else { - tracing::trace!(event_type = ?event.event, "Received unhandled event type"); - } - } - - tracing::trace!("SSE listener finished"); - tokio::time::sleep(tokio::time::Duration::from_secs(30)).await; - } - } - - /// Fetches an API request to the provided client URL like - /// `https://acme.com/api_endpoint/api_id` - /// - /// # Arguments - /// - /// * `client`: Prepopulated client please use `get_base_http_client` to construct this - /// * `cookie`: The komga cookie with the remember-me included - /// * `api_endpoint`: Endpoint which comes after the base_url doesn't require a - /// prepended `/` - /// * `api_id`: The ID of the object you are searching for added to the end of the - /// api_endpoint doesn't require a prepended `/` - /// - /// returns: This only preforms basic error handling on the json parsing - async fn fetch_api( - client: &reqwest::Client, - cookie: &str, - api_endpoint: &str, - api_id: &str, - ) -> Result { - client - .get(format!("{}/{}", api_endpoint, api_id)) - .header("Cookie", cookie) - .send() - .await? - .error_for_status()? - .json::() - .await - .map_err(|e| anyhow!("Failed to parse JSON: {}", e)) - } - - /// Finds the metadata provider and ID of the provided series - /// - /// # Arguments - /// - /// * `series`: The series object from which we want to grab the provider from. There - /// should be a links section which is populated with urls from which we - /// can extract the series ID. If not a simple search of the db for a manga - /// with the same title will be preformed - /// * `provider`: The preferred provider if this isn't available another will be used - /// in its place - /// * `db`: The metadata db connection - /// - /// returns: This contains the MediaSource and the ID of the series. - async fn find_provider_and_id( - series: &komga_series::Item, - provider: MediaSource, - db: &DatabaseConnection, - ) -> Result<(MediaSource, Option)> { - let providers = series.metadata.find_providers(); - if !providers.is_empty() { - Ok(providers - .iter() - .find(|x| x.0 == provider) - .cloned() - .or_else(|| providers.first().cloned()) - .unwrap_or_default()) - } else { - let db_manga = Metadata::find() - .filter(metadata::Column::Lot.eq(MediaLot::Manga)) - .filter(Expr::col(metadata::Column::Title).eq(&series.name)) - .one(db) - .await?; - - Ok(db_manga - .map(|manga| (manga.source, Some(manga.identifier))) - .unwrap_or_default()) - } - } - - fn calculate_percentage(curr_page: i32, total_page: i32) -> Decimal { - if total_page == 0 { - return Decimal::zero(); - } - - let percentage = (curr_page as f64 / total_page as f64) * 100.0; - - Decimal::from_f64(percentage).unwrap_or(Decimal::zero()) - } - - /// Processes the events which are provided by the receiver - /// - /// # Arguments - /// - /// * `base_url`: URL for komga - /// * `cookie`: The komga cookie with the remember-me included - /// * `provider`: The preferred provider if this isn't available another will be used - /// in its place - /// * `data`: The data from the event - /// - /// returns: If the event had no issues processing contains the media which was read - /// otherwise none - async fn process_events( - &self, - base_url: &str, - cookie: &str, - provider: MediaSource, - data: komga_events::Data, - ) -> Option { - let client = get_base_http_client(&format!("{}/api/v1/", base_url), None); - - let book: komga_book::Item = Self::fetch_api(&client, cookie, "books", &data.book_id) - .await - .ok()?; - let series: komga_series::Item = - Self::fetch_api(&client, cookie, "series", &book.series_id) - .await - .ok()?; - - let (source, id) = Self::find_provider_and_id(&series, provider, &self.db) - .await - .ok()?; - - let Some(id) = id else { - tracing::debug!( - "No MAL URL or database entry found for manga: {}", - series.name - ); - return None; - }; - - Some(IntegrationMediaSeen { - identifier: id, - lot: MediaLot::Manga, - source, - manga_chapter_number: Some(book.metadata.number.parse().unwrap_or_default()), - progress: Self::calculate_percentage(book.read_progress.page, book.media.pages_count), - provider_watched_on: Some("Komga".to_string()), - ..Default::default() - }) - } - - pub async fn komga_progress( - &self, - base_url: &str, - cookie: &str, - provider: MediaSource, - ) -> Result<(Vec, Vec)> { - // DEV: This object needs global lifetime so we can continue to use the receiver If - // we ever create more SSE Objects we may want to implement a higher level - // Controller or make a housekeeping function to make sure the background threads - // are running correctly and kill them when the app is killed (though rust should - // handle this). - static SSE_LISTS: KomgaEventHandler = KomgaEventHandler::new(); - - let mutex_receiver = SSE_LISTS.get_receiver(); - let mut receiver = { - let mut guard = mutex_receiver.lock().unwrap(); - guard.take() - }; - - if receiver.is_none() { - let (tx, rx) = mpsc::unbounded_channel::(); - receiver = Some(rx); - - let base_url = base_url.to_string(); - let cookie = cookie.to_string(); - let mutex_task = SSE_LISTS.get_task(); - - mutex_task.get_or_init(|| { - tokio::spawn(async move { - if let Err(e) = IntegrationService::sse_listener(tx, base_url, cookie).await { - tracing::error!("SSE listener error: {}", e); - } - }); - }); - } - - // Use hashmap here so we don't dupe pulls for a single book - let mut unique_media_items: HashMap = HashMap::new(); - - if let Some(mut recv) = receiver { - loop { - match recv.try_recv() { - Ok(event) => { - tracing::debug!("Received event {:?}", event); - match unique_media_items.entry(event.book_id.clone()) { - Entry::Vacant(entry) => { - if let Some(processed_event) = self - .process_events(base_url, cookie, provider, event.clone()) - .await - { - entry.insert(processed_event); - } else { - tracing::warn!( - "Failed to process event for book_id: {}", - event.book_id - ); - } - } - _ => continue, - } - } - Err(TryRecvError::Empty) => break, - Err(e) => return Err(anyhow::anyhow!("Receiver error: {}", e)), - } - } - - // Put the receiver back - mutex_receiver.lock().unwrap().replace(recv); - } - - let mut media_items: Vec = unique_media_items.into_values().collect(); - media_items.sort_by(|a, b| a.manga_chapter_number.cmp(&b.manga_chapter_number)); - tracing::debug!("Media Items: {:?}", media_items); - - Ok((media_items, vec![])) - } -} diff --git a/apps/backend/src/integrations/mod.rs b/apps/backend/src/integrations/mod.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/apps/backend/src/models.rs b/apps/backend/src/models.rs deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/crates/enums/src/lib.rs b/crates/enums/src/lib.rs index dd804c615a..826879457c 100644 --- a/crates/enums/src/lib.rs +++ b/crates/enums/src/lib.rs @@ -448,7 +448,6 @@ pub enum IntegrationLot { #[serde(rename_all = "snake_case")] pub enum IntegrationProvider { Audiobookshelf, - Komga, Jellyfin, Emby, Plex, diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index 2629be7b13..30003dc49b 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -29,45 +29,8 @@ use sonarr_api_rs::{ }, models::{AddSeriesOptions as SonarrAddSeriesOptions, SeriesResource as SonarrSeriesResource}, }; -<<<<<<<< HEAD:apps/backend/src/integrations/mod.rs - -use crate::{ - entities::{metadata, prelude::Metadata}, - models::{audiobookshelf_models, media::CommitMediaInput}, - providers::google_books::GoogleBooksService, - traits::TraceOk, - utils::{get_base_http_client, ilike_sql}, -}; - -mod komga; - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct IntegrationMediaSeen { - pub identifier: String, - pub lot: MediaLot, - #[serde(default)] - pub source: MediaSource, - pub progress: Decimal, - pub show_season_number: Option, - pub show_episode_number: Option, - pub podcast_episode_number: Option, - pub anime_episode_number: Option, - pub manga_chapter_number: Option, - pub manga_volume_number: Option, - pub provider_watched_on: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize, Default)] -pub struct IntegrationMediaCollection { - pub identifier: String, - pub lot: MediaLot, - pub source: MediaSource, - pub collection: String, -} -======== use specific_models::audiobookshelf as audiobookshelf_models; use traits::TraceOk; ->>>>>>>> refs/heads/main:crates/services/integration/src/lib.rs mod komga; From 5c9c4f029a17330517e7487054649d2504cae9e1 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:01:49 +0530 Subject: [PATCH 31/61] ci: revert ci changes --- .github/workflows/main.yml | 27 ++++----------------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 043bf17f02..ca80ca2196 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,34 +26,15 @@ jobs: steps: - uses: actions/checkout@v4 with: - fetch-depth: 0 - - name: Check maintainer status - uses: actions/github-script@v6 - id: check-maintainer - with: - github-token: ${{secrets.GITHUB_TOKEN}} - result-encoding: string - script: | - const { data: permission } = await github.rest.repos.getCollaboratorPermissionLevel({ - owner: context.repo.owner, - repo: context.repo.repo, - username: context.actor - }); - const hasPermission = (permission.permission === 'admin' || permission.permission === 'maintain') ? 'true' : 'false'; - console.log({ hasPermission }); - return hasPermission; - - name: Check commit message and run status + fetch-depth: 2 + - name: Check commit message id: check run: | if [[ "${{ github.event_name }}" == "push" ]]; then - if [[ "${{ steps.check-maintainer.outputs.result }}" == "true" ]]; then - echo "should-run=true" >> $GITHUB_OUTPUT - else - echo "should-run=false" >> $GITHUB_OUTPUT - fi + echo "should-run=true" >> $GITHUB_OUTPUT elif [[ "${{ github.event_name }}" == "pull_request" ]]; then COMMIT_MSG=$(git log --format=%B ${{ github.event.pull_request.head.sha }}) - if [[ "$COMMIT_MSG" == *"Run CI"* ]] && [[ "${{ steps.check-maintainer.outputs.result }}" == "true" ]]; then + if [[ "$COMMIT_MSG" == *"Run CI"* ]]; then echo "should-run=true" >> $GITHUB_OUTPUT else echo "should-run=false" >> $GITHUB_OUTPUT From 01f8d7fe71771b7a9c5e2561f8367235cd241994 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:09:05 +0530 Subject: [PATCH 32/61] chore(integrations): import structure --- crates/services/integration/src/komga.rs | 70 ++++++++++++++---------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/crates/services/integration/src/komga.rs b/crates/services/integration/src/komga.rs index ff2e40cb2d..477f8270ab 100644 --- a/crates/services/integration/src/komga.rs +++ b/crates/services/integration/src/komga.rs @@ -4,32 +4,25 @@ use std::{ }; use anyhow::{anyhow, Context, Result}; +use application_utils::get_base_http_client; use async_graphql::futures_util::StreamExt; +use database_models::{metadata, prelude::Metadata}; +use enums::{MediaLot, MediaSource}; use eventsource_stream::Eventsource; +use reqwest::Url; use rust_decimal::{ - Decimal, prelude::{FromPrimitive, Zero}, + Decimal, }; use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; use sea_query::Expr; -use serde::de::DeserializeOwned; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tokio::sync::{mpsc, mpsc::error::TryRecvError, mpsc::UnboundedReceiver}; -use database_models::{ - metadata, - prelude::Metadata -}; - -use crate::{ - get_base_http_client, - MediaLot, - MediaSource, -}; - use super::{IntegrationMediaCollection, IntegrationMediaSeen, IntegrationService}; mod komga_book { - use serde::{Deserialize, Serialize}; + use super::*; #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -71,11 +64,6 @@ mod komga_book { } mod komga_series { - use openidconnect::url::Url; - use serde::{Deserialize, Serialize}; - - use crate::MediaSource; - use super::*; #[derive(Debug, Serialize, Deserialize)] @@ -161,7 +149,7 @@ mod komga_series { } mod komga_events { - use serde::{Deserialize, Serialize}; + use super::*; #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] @@ -354,13 +342,24 @@ impl IntegrationService { ) -> Option { let client = get_base_http_client(&format!("{}/api/v1/", base_url), None); - let book: komga_book::Item = Self::fetch_api(&client, komga_username, komga_password, "books", &data.book_id) - .await - .ok()?; - let series: komga_series::Item = - Self::fetch_api(&client, komga_username, komga_password, "series", &book.series_id) - .await - .ok()?; + let book: komga_book::Item = Self::fetch_api( + &client, + komga_username, + komga_password, + "books", + &data.book_id, + ) + .await + .ok()?; + let series: komga_series::Item = Self::fetch_api( + &client, + komga_username, + komga_password, + "series", + &book.series_id, + ) + .await + .ok()?; let (source, id) = Self::find_provider_and_id(&series, provider, &self.db) .await @@ -416,7 +415,14 @@ impl IntegrationService { mutex_task.get_or_init(|| { tokio::spawn(async move { - if let Err(e) = IntegrationService::sse_listener(tx, base_url, komga_username, komga_password).await { + if let Err(e) = IntegrationService::sse_listener( + tx, + base_url, + komga_username, + komga_password, + ) + .await + { tracing::error!("SSE listener error: {}", e); } }); @@ -434,7 +440,13 @@ impl IntegrationService { match unique_media_items.entry(event.book_id.clone()) { Entry::Vacant(entry) => { if let Some(processed_event) = self - .process_events(base_url, komga_username, komga_password, provider, event.clone()) + .process_events( + base_url, + komga_username, + komga_password, + provider, + event.clone(), + ) .await { entry.insert(processed_event); From f2397b10ef7d5667127a4374c568beacbe208088 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:09:18 +0530 Subject: [PATCH 33/61] build(integrations): remove uneeded package --- Cargo.lock | 1 - crates/services/integration/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b9ceab77d..2ba6c04694 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3175,7 +3175,6 @@ dependencies = [ "enums", "eventsource-stream", "media-models", - "openidconnect", "providers", "radarr-api-rs", "regex", diff --git a/crates/services/integration/Cargo.toml b/crates/services/integration/Cargo.toml index faf60b36e0..82e317db03 100644 --- a/crates/services/integration/Cargo.toml +++ b/crates/services/integration/Cargo.toml @@ -12,7 +12,6 @@ eventsource-stream = "=0.2.3" database-models = { path = "../../models/database" } database-utils = { path = "../../utils/database" } media-models = { path = "../../models/media" } -openidconnect = { workspace = true } providers = { path = "../../providers" } radarr-api-rs = "=3.0.1" regex = { workspace = true } From cc8178d422e41055bf00684ce183767d6a4e55a6 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:20:55 +0530 Subject: [PATCH 34/61] ci: change computation of image names Run CI. --- .github/workflows/main.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ca80ca2196..3af0c36dc5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,6 +22,7 @@ jobs: outputs: should-run: ${{ steps.check.outputs.should-run }} should-release: ${{ steps.tag.outputs.should-release }} + image-names: ${{ steps.check.outputs.image-names }} steps: - uses: actions/checkout@v4 @@ -30,6 +31,11 @@ jobs: - name: Check commit message id: check run: | + read -r -d '' IMAGE_NAMES << EOM + name=${{ secrets.DOCKER_USERNAME }}/${{ github.event.repository.name }} + name=${{ env.GHCR_REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }} + EOM + if [[ "${{ github.event_name }}" == "push" ]]; then echo "should-run=true" >> $GITHUB_OUTPUT elif [[ "${{ github.event_name }}" == "pull_request" ]]; then @@ -42,6 +48,8 @@ jobs: else echo "should-run=false" >> $GITHUB_OUTPUT fi + + echo "image-names=$IMAGE_NAMES" >> $GITHUB_OUTPUT - name: Check tag id: tag run: | @@ -200,9 +208,7 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: | - name=${{ secrets.DOCKER_USERNAME }}/${{ github.event.repository.name }} - name=${{ env.GHCR_REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }} + images: ${{ needs.pre-workflow-checks.outputs.image-names }} tags: | type=ref,event=pr type=raw,value=develop,enable={{is_default_branch}} From 36bd0e3390a298aae53f1ae827cb9847212110b1 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:23:40 +0530 Subject: [PATCH 35/61] ci: hardcode docker username Run CI. --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3af0c36dc5..d9fd75c9b1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -32,8 +32,8 @@ jobs: id: check run: | read -r -d '' IMAGE_NAMES << EOM - name=${{ secrets.DOCKER_USERNAME }}/${{ github.event.repository.name }} - name=${{ env.GHCR_REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }} + name=ignisda/${{ github.event.repository.name }} + name=${{ env.GHCR_REGISTRY }}/ignisda/${{ github.event.repository.name }} EOM if [[ "${{ github.event_name }}" == "push" ]]; then @@ -202,7 +202,7 @@ jobs: - name: Log in to the docker hub container registry uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} + username: ignisda password: ${{ secrets.DOCKER_TOKEN }} - name: Extract metadata for Docker id: meta From 3180b9bdf7cab77ee83b76847b5f02d93c2f508a Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:25:21 +0530 Subject: [PATCH 36/61] ci: enable hardmode Run CI. --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d9fd75c9b1..7ff405ce81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,6 +31,8 @@ jobs: - name: Check commit message id: check run: | + set -euxo pipefail + read -r -d '' IMAGE_NAMES << EOM name=ignisda/${{ github.event.repository.name }} name=${{ env.GHCR_REGISTRY }}/ignisda/${{ github.event.repository.name }} From 9b37a40a2ec001df8761c996382ad0e0b58b7124 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:26:41 +0530 Subject: [PATCH 37/61] ci: quote names Run CI. --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7ff405ce81..21500e1b4e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,8 +34,8 @@ jobs: set -euxo pipefail read -r -d '' IMAGE_NAMES << EOM - name=ignisda/${{ github.event.repository.name }} - name=${{ env.GHCR_REGISTRY }}/ignisda/${{ github.event.repository.name }} + name="ignisda/${{ github.event.repository.name }}" + name="${{ env.GHCR_REGISTRY }}/ignisda/${{ github.event.repository.name }}" EOM if [[ "${{ github.event_name }}" == "push" ]]; then From 6f6c2c9e31d29658f7537dcc7fab6ee99f551acb Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:30:46 +0530 Subject: [PATCH 38/61] ci: no heredoc fuckery Run CI. --- .github/workflows/main.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21500e1b4e..0e713ac8d6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -31,12 +31,10 @@ jobs: - name: Check commit message id: check run: | - set -euxo pipefail - - read -r -d '' IMAGE_NAMES << EOM - name="ignisda/${{ github.event.repository.name }}" - name="${{ env.GHCR_REGISTRY }}/ignisda/${{ github.event.repository.name }}" - EOM + IMAGE_NAMES=" + name=ignisda/${{ github.event.repository.name }} + name=${{ env.GHCR_REGISTRY }}/ignisda/${{ github.event.repository.name }} + " if [[ "${{ github.event_name }}" == "push" ]]; then echo "should-run=true" >> $GITHUB_OUTPUT From e3e09052bba484ace06518c94ff2f7fe85a85d60 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:32:40 +0530 Subject: [PATCH 39/61] ci: change name of image Run CI. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0e713ac8d6..f7aa8e6a63 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,7 +33,7 @@ jobs: run: | IMAGE_NAMES=" name=ignisda/${{ github.event.repository.name }} - name=${{ env.GHCR_REGISTRY }}/ignisda/${{ github.event.repository.name }} + name=${{ env.GHCR_REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }} " if [[ "${{ github.event_name }}" == "push" ]]; then From dd54876270e70686e9b45b6ba5ffdb1b3b1e8225 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:40:36 +0530 Subject: [PATCH 40/61] ci: remove docker for forks Run CI. --- .github/workflows/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f7aa8e6a63..c1aa83c5de 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,6 +42,10 @@ jobs: COMMIT_MSG=$(git log --format=%B ${{ github.event.pull_request.head.sha }}) if [[ "$COMMIT_MSG" == *"Run CI"* ]]; then echo "should-run=true" >> $GITHUB_OUTPUT + # if it is a forked PR, we need to remove the first line of the IMAGE_NAMES + if [[ "${{ github.event.pull_request.head.repo.fork }}" ]]; then + IMAGE_NAMES=$(echo "$IMAGE_NAMES" | sed '1d') + fi else echo "should-run=false" >> $GITHUB_OUTPUT fi @@ -49,6 +53,7 @@ jobs: echo "should-run=false" >> $GITHUB_OUTPUT fi + echo "image-names=$IMAGE_NAMES" echo "image-names=$IMAGE_NAMES" >> $GITHUB_OUTPUT - name: Check tag id: tag From 12cae01ac7f4383822686d5e6e48733d99c4b9b7 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:42:22 +0530 Subject: [PATCH 41/61] ci: update condition Run CI. --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c1aa83c5de..a6f72a1733 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,7 +43,7 @@ jobs: if [[ "$COMMIT_MSG" == *"Run CI"* ]]; then echo "should-run=true" >> $GITHUB_OUTPUT # if it is a forked PR, we need to remove the first line of the IMAGE_NAMES - if [[ "${{ github.event.pull_request.head.repo.fork }}" ]]; then + if [[ "${{ github.event.pull_request.head.repo.full_name == github.repository }}" ]]; then IMAGE_NAMES=$(echo "$IMAGE_NAMES" | sed '1d') fi else @@ -200,12 +200,14 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Log in to the ghcr container registry uses: docker/login-action@v3 + continue-on-error: true with: registry: ${{ env.GHCR_REGISTRY }} username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to the docker hub container registry uses: docker/login-action@v3 + continue-on-error: true with: username: ignisda password: ${{ secrets.DOCKER_TOKEN }} From eccf2c4c727f24dfc6a1097fb887a13e63b914a4 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:43:18 +0530 Subject: [PATCH 42/61] ci: logging Run CI. --- .github/workflows/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a6f72a1733..5e785cb20e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,6 +36,9 @@ jobs: name=${{ env.GHCR_REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }} " + echo "${{ github.event.pull_request.head.repo.full_name }}" + echo "${{ github.repository }}" + if [[ "${{ github.event_name }}" == "push" ]]; then echo "should-run=true" >> $GITHUB_OUTPUT elif [[ "${{ github.event_name }}" == "pull_request" ]]; then From 6debb41572e3d388b471fbdb7d5b53ae859b3b8b Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:53:33 +0530 Subject: [PATCH 43/61] ci: use gh actions script Run CI. --- .github/workflows/main.yml | 79 ++++++++++++++++++++------------------ 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5e785cb20e..8994ce0e45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,52 +20,55 @@ jobs: pre-workflow-checks: runs-on: ubuntu-latest outputs: - should-run: ${{ steps.check.outputs.should-run }} - should-release: ${{ steps.tag.outputs.should-release }} - image-names: ${{ steps.check.outputs.image-names }} + should-run: ${{ steps.set_outputs.outputs.should-run }} + image-names: ${{ steps.set_outputs.outputs.image-names }} + should-release: ${{ steps.set_outputs.outputs.should-release }} steps: - uses: actions/checkout@v4 with: fetch-depth: 2 - - name: Check commit message - id: check - run: | - IMAGE_NAMES=" - name=ignisda/${{ github.event.repository.name }} - name=${{ env.GHCR_REGISTRY }}/${{ github.repository_owner }}/${{ github.event.repository.name }} - " + - name: Set outputs + id: set_outputs + uses: actions/github-script@v7 + with: + script: | + const repositoryName = context.payload.repository.name; + const owner = context.repo.owner; + const ghcrRegistry = process.env.GHCR_REGISTRY; - echo "${{ github.event.pull_request.head.repo.full_name }}" - echo "${{ github.repository }}" + let imageNames = [ + `name=ignisda/${repositoryName}`, + `name=${ghcrRegistry}/${owner}/${repositoryName}` + ]; - if [[ "${{ github.event_name }}" == "push" ]]; then - echo "should-run=true" >> $GITHUB_OUTPUT - elif [[ "${{ github.event_name }}" == "pull_request" ]]; then - COMMIT_MSG=$(git log --format=%B ${{ github.event.pull_request.head.sha }}) - if [[ "$COMMIT_MSG" == *"Run CI"* ]]; then - echo "should-run=true" >> $GITHUB_OUTPUT - # if it is a forked PR, we need to remove the first line of the IMAGE_NAMES - if [[ "${{ github.event.pull_request.head.repo.full_name == github.repository }}" ]]; then - IMAGE_NAMES=$(echo "$IMAGE_NAMES" | sed '1d') - fi - else - echo "should-run=false" >> $GITHUB_OUTPUT - fi - else - echo "should-run=false" >> $GITHUB_OUTPUT - fi + if (context.eventName === "push") { + core.setOutput('should-run', 'true'); + } else if (context.eventName === "pull_request") { + const commitMsg = await github.rest.repos.getCommit({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: context.payload.pull_request.head.sha + }).then(commit => commit.data.commit.message); + if (commitMsg.includes("Run CI")) { + core.setOutput('should-run', 'true'); + if (context.payload.pull_request.head.repo.full_name !== context.repo.full_name) { + imageNames.shift(); + } + } else { + core.setOutput('should-run', 'false'); + } + } else { + core.setOutput('should-run', 'false'); + } - echo "image-names=$IMAGE_NAMES" - echo "image-names=$IMAGE_NAMES" >> $GITHUB_OUTPUT - - name: Check tag - id: tag - run: | - if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/tags/${{ github.ref_name }}" ]]; then - echo "should-release=true" >> $GITHUB_OUTPUT - else - echo "should-release=false" >> $GITHUB_OUTPUT - fi + if (context.eventName === "push" && context.ref.startsWith("refs/tags/")) { + core.setOutput('should-release', 'true'); + } else { + core.setOutput('should-release', 'false'); + } + + core.setOutput('image-names', imageNames.join('\n')); create-release: needs: From a9d23bbbd6a8144893bbf3136f73b3a692e1c656 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 10:54:59 +0530 Subject: [PATCH 44/61] ci: print all outputs Run CI. --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8994ce0e45..8d8076b01c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,6 +69,8 @@ jobs: } core.setOutput('image-names', imageNames.join('\n')); + - name: Print all outputs + run: cat $GITHUB_OUTPUT create-release: needs: From ea960dc00ff067f22fb515217f645711c4a1d3f2 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 11:14:28 +0530 Subject: [PATCH 45/61] ci: use correct image names Run CI. --- .github/workflows/main.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8d8076b01c..3f61c57b9c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,11 +34,12 @@ jobs: with: script: | const repositoryName = context.payload.repository.name; - const owner = context.repo.owner; + const owner = context.actor; const ghcrRegistry = process.env.GHCR_REGISTRY; + const dockerUsername = process.env.DOCKER_USERNAME; let imageNames = [ - `name=ignisda/${repositoryName}`, + `name=${dockerUsername}/${repositoryName}`, `name=${ghcrRegistry}/${owner}/${repositoryName}` ]; @@ -69,8 +70,6 @@ jobs: } core.setOutput('image-names', imageNames.join('\n')); - - name: Print all outputs - run: cat $GITHUB_OUTPUT create-release: needs: @@ -211,13 +210,13 @@ jobs: continue-on-error: true with: registry: ${{ env.GHCR_REGISTRY }} - username: ${{ github.repository_owner }} + username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to the docker hub container registry uses: docker/login-action@v3 continue-on-error: true with: - username: ignisda + username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} - name: Extract metadata for Docker id: meta From c41b36537bdafb5d5b20fe16c2f0e7635c738b2b Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sat, 17 Aug 2024 11:32:12 +0530 Subject: [PATCH 46/61] ci: use correct owner name --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3f61c57b9c..52464a6514 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: with: script: | const repositoryName = context.payload.repository.name; - const owner = context.actor; + const owner = context.repo.owner; const ghcrRegistry = process.env.GHCR_REGISTRY; const dockerUsername = process.env.DOCKER_USERNAME; @@ -210,7 +210,7 @@ jobs: continue-on-error: true with: registry: ${{ env.GHCR_REGISTRY }} - username: ${{ github.actor }} + username: ${{ github.repository_owner }} password: ${{ secrets.GITHUB_TOKEN }} - name: Log in to the docker hub container registry uses: docker/login-action@v3 From ba08f712e8bf544c0ad11814264d8e9dfc75dc5f Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Fri, 16 Aug 2024 23:38:06 -0700 Subject: [PATCH 47/61] Upgraded to a trait model --- .../services/integration/src/integration.rs | 8 ++ .../integration/src/integration_type.rs | 5 + crates/services/integration/src/komga.rs | 117 ++++++++---------- crates/services/integration/src/lib.rs | 19 +++ crates/services/miscellaneous/src/lib.rs | 16 ++- 5 files changed, 94 insertions(+), 71 deletions(-) create mode 100644 crates/services/integration/src/integration.rs create mode 100644 crates/services/integration/src/integration_type.rs diff --git a/crates/services/integration/src/integration.rs b/crates/services/integration/src/integration.rs new file mode 100644 index 0000000000..eb313da065 --- /dev/null +++ b/crates/services/integration/src/integration.rs @@ -0,0 +1,8 @@ +use anyhow::Result; + +use crate::IntegrationMediaCollection; +use crate::IntegrationMediaSeen; + +pub trait Integration { + async fn progress(&self) -> Result<(Vec, Vec)>; +} diff --git a/crates/services/integration/src/integration_type.rs b/crates/services/integration/src/integration_type.rs new file mode 100644 index 0000000000..b63701100e --- /dev/null +++ b/crates/services/integration/src/integration_type.rs @@ -0,0 +1,5 @@ +use enums::MediaSource; + +pub enum IntegrationType { + Komga(String, String, String, MediaSource), +} diff --git a/crates/services/integration/src/komga.rs b/crates/services/integration/src/komga.rs index 477f8270ab..890ad8f9ea 100644 --- a/crates/services/integration/src/komga.rs +++ b/crates/services/integration/src/komga.rs @@ -4,22 +4,25 @@ use std::{ }; use anyhow::{anyhow, Context, Result}; -use application_utils::get_base_http_client; use async_graphql::futures_util::StreamExt; -use database_models::{metadata, prelude::Metadata}; -use enums::{MediaLot, MediaSource}; use eventsource_stream::Eventsource; use reqwest::Url; use rust_decimal::{ - prelude::{FromPrimitive, Zero}, Decimal, + prelude::{FromPrimitive, Zero}, }; use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; use sea_query::Expr; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tokio::sync::{mpsc, mpsc::error::TryRecvError, mpsc::UnboundedReceiver}; -use super::{IntegrationMediaCollection, IntegrationMediaSeen, IntegrationService}; +use application_utils::get_base_http_client; +use database_models::{metadata, prelude::Metadata}; +use enums::{MediaLot, MediaSource}; + +use crate::integration::Integration; + +use super::{IntegrationMediaCollection, IntegrationMediaSeen}; mod komga_book { use super::*; @@ -181,7 +184,25 @@ impl KomgaEventHandler { } } -impl IntegrationService { +pub struct KomgaIntegration { + base_url: String, + username: String, + password: String, + provider: MediaSource, + db: DatabaseConnection, +} + +impl KomgaIntegration { + pub fn new(base_url: String, username: String, password: String, provider: MediaSource, db: DatabaseConnection) -> Self { + Self { + base_url, + username, + password, + provider, + db, + } + } + /// Generates the sse listener for komga. This is intended to be run from another /// thread if you run this in the main thread it will lock it up /// @@ -190,7 +211,8 @@ impl IntegrationService { /// * `sender`: The unbounded sender, lifetime of this sender is the lifetime of this /// function so the sender doesn't need global lifetime /// * `base_url`: URL for komga - /// * `cookie`: The komga cookie with the remember-me included + /// * `komga_username`: The komga username + /// * `komga_password`: The komga password /// /// returns: Never Returns async fn sse_listener( @@ -254,15 +276,14 @@ impl IntegrationService { /// /// returns: This only preforms basic error handling on the json parsing async fn fetch_api( + &self, client: &reqwest::Client, - komga_username: &str, - komga_password: &str, api_endpoint: &str, api_id: &str, ) -> Result { client .get(format!("{}/{}", api_endpoint, api_id)) - .basic_auth(komga_username, Some(komga_password)) + .basic_auth(self.username.clone(), Some(self.password.clone())) .send() .await? .error_for_status()? @@ -279,21 +300,17 @@ impl IntegrationService { /// should be a links section which is populated with urls from which we /// can extract the series ID. If not a simple search of the db for a manga /// with the same title will be preformed - /// * `provider`: The preferred provider if this isn't available another will be used - /// in its place - /// * `db`: The metadata db connection /// /// returns: This contains the MediaSource and the ID of the series. async fn find_provider_and_id( + &self, series: &komga_series::Item, - provider: MediaSource, - db: &DatabaseConnection, ) -> Result<(MediaSource, Option)> { let providers = series.metadata.find_providers(); if !providers.is_empty() { Ok(providers .iter() - .find(|x| x.0 == provider) + .find(|x| x.0 == self.provider) .cloned() .or_else(|| providers.first().cloned()) .unwrap_or_default()) @@ -301,7 +318,7 @@ impl IntegrationService { let db_manga = Metadata::find() .filter(metadata::Column::Lot.eq(MediaLot::Manga)) .filter(Expr::col(metadata::Column::Title).eq(&series.name)) - .one(db) + .one(&self.db) .await?; Ok(db_manga @@ -320,48 +337,28 @@ impl IntegrationService { Decimal::from_f64(percentage).unwrap_or(Decimal::zero()) } - /// Processes the events which are provided by the receiver - /// - /// # Arguments - /// - /// * `base_url`: URL for komga - /// * `cookie`: The komga cookie with the remember-me included - /// * `provider`: The preferred provider if this isn't available another will be used - /// in its place - /// * `data`: The data from the event - /// - /// returns: If the event had no issues processing contains the media which was read - /// otherwise none async fn process_events( &self, - base_url: &str, - komga_username: &str, - komga_password: &str, - provider: MediaSource, data: komga_events::Data, ) -> Option { - let client = get_base_http_client(&format!("{}/api/v1/", base_url), None); + let client = get_base_http_client(&format!("{}/api/v1/", self.base_url), None); - let book: komga_book::Item = Self::fetch_api( + let book: komga_book::Item = self.fetch_api( &client, - komga_username, - komga_password, "books", &data.book_id, ) - .await - .ok()?; - let series: komga_series::Item = Self::fetch_api( + .await + .ok()?; + let series: komga_series::Item = self.fetch_api( &client, - komga_username, - komga_password, "series", &book.series_id, ) - .await - .ok()?; + .await + .ok()?; - let (source, id) = Self::find_provider_and_id(&series, provider, &self.db) + let (source, id) = self.find_provider_and_id(&series) .await .ok()?; @@ -385,11 +382,7 @@ impl IntegrationService { } pub async fn komga_progress( - &self, - base_url: &str, - komga_username: &str, - komga_password: &str, - provider: MediaSource, + &self ) -> Result<(Vec, Vec)> { // DEV: This object needs global lifetime so we can continue to use the receiver If // we ever create more SSE Objects we may want to implement a higher level @@ -408,20 +401,20 @@ impl IntegrationService { let (tx, rx) = mpsc::unbounded_channel::(); receiver = Some(rx); - let base_url = base_url.to_string(); + let base_url = self.base_url.to_string(); let mutex_task = SSE_LISTS.get_task(); - let komga_username = komga_username.to_string(); - let komga_password = komga_password.to_string(); + let komga_username = self.username.to_string(); + let komga_password = self.password.to_string(); mutex_task.get_or_init(|| { tokio::spawn(async move { - if let Err(e) = IntegrationService::sse_listener( + if let Err(e) = Self::sse_listener( tx, base_url, komga_username, komga_password, ) - .await + .await { tracing::error!("SSE listener error: {}", e); } @@ -440,13 +433,7 @@ impl IntegrationService { match unique_media_items.entry(event.book_id.clone()) { Entry::Vacant(entry) => { if let Some(processed_event) = self - .process_events( - base_url, - komga_username, - komga_password, - provider, - event.clone(), - ) + .process_events(event.clone()) .await { entry.insert(processed_event); @@ -476,3 +463,9 @@ impl IntegrationService { Ok((media_items, vec![])) } } + +impl Integration for KomgaIntegration { + async fn progress(&self) -> Result<(Vec, Vec)> { + self.komga_progress().await + } +} diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index 30003dc49b..f73af104dc 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -32,6 +32,15 @@ use sonarr_api_rs::{ use specific_models::audiobookshelf as audiobookshelf_models; use traits::TraceOk; +use crate::{ + integration::Integration, + integration_type::IntegrationType, + komga::KomgaIntegration +}; + +pub mod integration_type; +mod integration; + mod komga; #[derive(Debug)] @@ -569,4 +578,14 @@ impl IntegrationService { .trace_ok(); Ok(()) } + + pub async fn process_progress(&self, integration_type: IntegrationType) + -> Result<(Vec, Vec)> { + match integration_type { + IntegrationType::Komga(base_url, username, password, provider) => { + let komga = KomgaIntegration::new(base_url, username, password, provider, self.db.clone()); + komga.progress().await + } + } + } } diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index d689cd721b..7a4cbc4ab2 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -56,7 +56,7 @@ use enums::{ use file_storage_service::FileStorageService; use fitness_models::UserUnitSystem; use futures::TryStreamExt; -use integration_service::IntegrationService; +use integration_service::{IntegrationService, integration_type::IntegrationType}; use itertools::Itertools; use jwt_service::sign; use markdown::{to_html_with_options as markdown_to_html_opts, CompileOptions, Options}; @@ -4444,14 +4444,12 @@ impl MiscellaneousService { } IntegrationProvider::Komga => { let specifics = integration.clone().provider_specifics.unwrap(); - integration_service - .komga_progress( - &specifics.komga_base_url.unwrap(), - &specifics.komga_username.unwrap(), - &specifics.komga_password.unwrap(), - specifics.komga_provider.unwrap(), - ) - .await + integration_service.process_progress(IntegrationType::Komga( + specifics.komga_base_url.unwrap(), + specifics.komga_username.unwrap(), + specifics.komga_password.unwrap(), + specifics.komga_provider.unwrap(), + )).await } _ => continue, }; From fe023cd2cb5600b3f02ae8bc988238068da40c08 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 14:23:42 -0700 Subject: [PATCH 48/61] Migrated jellyfin to trait --- .../integration/src/integration_type.rs | 2 + crates/services/integration/src/jellyfin.rs | 101 ++++++++++++++++++ crates/services/integration/src/lib.rs | 83 ++------------ crates/services/miscellaneous/src/lib.rs | 27 +++-- 4 files changed, 125 insertions(+), 88 deletions(-) create mode 100644 crates/services/integration/src/jellyfin.rs diff --git a/crates/services/integration/src/integration_type.rs b/crates/services/integration/src/integration_type.rs index b63701100e..afc05afa1c 100644 --- a/crates/services/integration/src/integration_type.rs +++ b/crates/services/integration/src/integration_type.rs @@ -2,4 +2,6 @@ use enums::MediaSource; pub enum IntegrationType { Komga(String, String, String, MediaSource), + Jellyfin(String), + } diff --git a/crates/services/integration/src/jellyfin.rs b/crates/services/integration/src/jellyfin.rs new file mode 100644 index 0000000000..b15eb05e64 --- /dev/null +++ b/crates/services/integration/src/jellyfin.rs @@ -0,0 +1,101 @@ +use anyhow::bail; +use anyhow::Result; +use rust_decimal_macros::dec; + +use enums::{MediaLot, MediaSource}; +use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; + +use crate::integration::Integration; + +mod models { + use rust_decimal::Decimal; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct JellyfinWebhookSessionPlayStatePayload { + pub position_ticks: Option, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct JellyfinWebhookSessionPayload { + pub play_state: JellyfinWebhookSessionPlayStatePayload, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct JellyfinWebhookItemProviderIdsPayload { + pub tmdb: Option, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct JellyfinWebhookItemPayload { + pub run_time_ticks: Option, + #[serde(rename = "Type")] + pub item_type: String, + pub provider_ids: JellyfinWebhookItemProviderIdsPayload, + #[serde(rename = "ParentIndexNumber")] + pub season_number: Option, + #[serde(rename = "IndexNumber")] + pub episode_number: Option, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct JellyfinWebhookPayload { + pub event: Option, + pub item: JellyfinWebhookItemPayload, + pub series: Option, + pub session: JellyfinWebhookSessionPayload, + } +} + +pub struct JellyfinIntegration +{ + payload: String, +} + +impl JellyfinIntegration { + pub const fn new(payload: String) -> Self { + Self { + payload + } + } + + async fn jellyfin_progress( + &self + ) -> Result<(Vec, Vec)> { + let payload = serde_json::from_str::(&self.payload)?; + let identifier = payload.item.provider_ids.tmdb.as_ref() + .or_else(|| payload.series.as_ref().and_then(|s| s.provider_ids.tmdb.as_ref())) + .ok_or_else(|| anyhow::anyhow!("No TMDb ID associated with this media"))? + .clone(); + + let runtime = payload.item.run_time_ticks + .ok_or_else(|| anyhow::anyhow!("No run time associated with this media"))?; + + let position = payload.session.play_state.position_ticks + .ok_or_else(|| anyhow::anyhow!("No position associated with this media"))?; + + let lot = match payload.item.item_type.as_str() { + "Episode" => MediaLot::Show, + "Movie" => MediaLot::Movie, + _ => bail!("Only movies and shows supported"), + }; + + Ok((vec![IntegrationMediaSeen { + identifier, + lot, + source: MediaSource::Tmdb, + progress: position / runtime * dec!(100), + show_season_number: payload.item.season_number, + show_episode_number: payload.item.episode_number, + provider_watched_on: Some("Jellyfin".to_string()), + ..Default::default() + }], vec![])) + } +} + +impl Integration for JellyfinIntegration { + async fn progress(&self) -> Result<(Vec, Vec)> { + self.jellyfin_progress().await + } +} diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index f73af104dc..5378da2dd9 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -37,11 +37,13 @@ use crate::{ integration_type::IntegrationType, komga::KomgaIntegration }; +use crate::jellyfin::JellyfinIntegration; pub mod integration_type; mod integration; mod komga; +mod jellyfin; #[derive(Debug)] pub struct IntegrationService { @@ -85,84 +87,7 @@ impl IntegrationService { } } - pub async fn jellyfin_progress(&self, payload: &str) -> Result { - mod models { - use super::*; - - #[derive(Serialize, Deserialize, Debug, Clone)] - #[serde(rename_all = "PascalCase")] - pub struct JellyfinWebhookSessionPlayStatePayload { - pub position_ticks: Option, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - #[serde(rename_all = "PascalCase")] - pub struct JellyfinWebhookSessionPayload { - pub play_state: JellyfinWebhookSessionPlayStatePayload, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - #[serde(rename_all = "PascalCase")] - pub struct JellyfinWebhookItemProviderIdsPayload { - pub tmdb: Option, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - #[serde(rename_all = "PascalCase")] - pub struct JellyfinWebhookItemPayload { - pub run_time_ticks: Option, - #[serde(rename = "Type")] - pub item_type: String, - pub provider_ids: JellyfinWebhookItemProviderIdsPayload, - #[serde(rename = "ParentIndexNumber")] - pub season_number: Option, - #[serde(rename = "IndexNumber")] - pub episode_number: Option, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - #[serde(rename_all = "PascalCase")] - pub struct JellyfinWebhookPayload { - pub event: Option, - pub item: JellyfinWebhookItemPayload, - pub series: Option, - pub session: JellyfinWebhookSessionPayload, - } - } - let payload = serde_json::from_str::(payload)?; - let identifier = if let Some(id) = payload.item.provider_ids.tmdb.as_ref() { - Some(id.clone()) - } else { - payload - .series - .as_ref() - .and_then(|s| s.provider_ids.tmdb.clone()) - }; - if identifier.is_none() { - bail!("No TMDb ID associated with this media") - } - if payload.item.run_time_ticks.is_none() { - bail!("No run time associated with this media") - } - if payload.session.play_state.position_ticks.is_none() { - bail!("No position associated with this media") - } - let identifier = identifier.unwrap(); - let runtime = payload.item.run_time_ticks.unwrap(); - let position = payload.session.play_state.position_ticks.unwrap(); - let lot = match payload.item.item_type.as_str() { - "Episode" => MediaLot::Show, - "Movie" => MediaLot::Movie, - _ => bail!("Only movies and shows supported"), - }; - Ok(IntegrationMediaSeen { - identifier, - lot, - source: MediaSource::Tmdb, - progress: position / runtime * dec!(100), - show_season_number: payload.item.season_number, - show_episode_number: payload.item.episode_number, - provider_watched_on: Some("Jellyfin".to_string()), - ..Default::default() - }) - } pub async fn plex_progress( &self, @@ -586,6 +511,10 @@ impl IntegrationService { let komga = KomgaIntegration::new(base_url, username, password, provider, self.db.clone()); komga.progress().await } + IntegrationType::Jellyfin(payload) => { + let jellyfin = JellyfinIntegration::new(payload); + jellyfin.progress().await + } } } } diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index 7a4cbc4ab2..27b0da62f8 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -4651,21 +4651,26 @@ impl MiscellaneousService { } let service = self.get_integration_service(); let maybe_progress_update = match integration.provider { - IntegrationProvider::Kodi => service.kodi_progress(&payload).await, - IntegrationProvider::Emby => service.emby_progress(&payload).await, - IntegrationProvider::Jellyfin => service.jellyfin_progress(&payload).await, - IntegrationProvider::Plex => { - let specifics = integration.clone().provider_specifics.unwrap(); - service - .plex_progress(&payload, specifics.plex_username) - .await - } + // IntegrationProvider::Kodi => service.kodi_progress(&payload).await, + // IntegrationProvider::Emby => service.emby_progress(&payload).await, + IntegrationProvider::Jellyfin => service.process_progress(IntegrationType::Jellyfin( + payload.clone() + )).await, + // IntegrationProvider::Plex => { + // let specifics = integration.clone().provider_specifics.unwrap(); + // service + // .plex_progress(&payload, specifics.plex_username) + // .await + // } _ => return Err(Error::new("Unsupported integration source".to_owned())), }; match maybe_progress_update { Ok(pu) => { - self.integration_progress_update(&integration, pu, &integration.user_id) - .await?; + let media_vec = pu.0; + for media in media_vec + { + self.integration_progress_update(&integration, media.clone(), &integration.user_id).await?; + } let mut to_update: integration::ActiveModel = integration.into(); to_update.last_triggered_on = ActiveValue::Set(Some(Utc::now())); to_update.update(&self.db).await?; From 2834ac2fa48e9d89380aa135e0009eaa7f4ce729 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 14:26:12 -0700 Subject: [PATCH 49/61] Fixed visibility of function --- crates/services/integration/src/komga.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/services/integration/src/komga.rs b/crates/services/integration/src/komga.rs index 890ad8f9ea..e93cac39f7 100644 --- a/crates/services/integration/src/komga.rs +++ b/crates/services/integration/src/komga.rs @@ -381,7 +381,7 @@ impl KomgaIntegration { }) } - pub async fn komga_progress( + async fn komga_progress( &self ) -> Result<(Vec, Vec)> { // DEV: This object needs global lifetime so we can continue to use the receiver If From 0f528a2ae53c398f9391d4b6f4a4031dfd094148 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 14:49:03 -0700 Subject: [PATCH 50/61] Emby Integration Migration --- crates/services/integration/src/emby.rs | 113 ++++++++++++++++++ .../integration/src/integration_type.rs | 2 +- crates/services/integration/src/lib.rs | 106 ++-------------- .../integration/src/show_identifier.rs | 43 +++++++ crates/services/miscellaneous/src/lib.rs | 4 +- 5 files changed, 167 insertions(+), 101 deletions(-) create mode 100644 crates/services/integration/src/emby.rs create mode 100644 crates/services/integration/src/show_identifier.rs diff --git a/crates/services/integration/src/emby.rs b/crates/services/integration/src/emby.rs new file mode 100644 index 0000000000..4d1d649e24 --- /dev/null +++ b/crates/services/integration/src/emby.rs @@ -0,0 +1,113 @@ +use rust_decimal_macros::dec; +use sea_orm::DatabaseConnection; +use sea_orm::prelude::async_trait::async_trait; +use enums::{MediaLot, MediaSource}; +use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; +use crate::integration::Integration; +use crate::show_identifier::ShowIdentifier; + +mod models { + use rust_decimal::Decimal; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct EmbyWebhookPlaybackInfoPayload { + pub position_ticks: Option, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct EmbyWebhookItemProviderIdsPayload { + pub tmdb: Option, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct EmbyWebhookItemPayload { + pub run_time_ticks: Option, + #[serde(rename = "Type")] + pub item_type: String, + pub provider_ids: EmbyWebhookItemProviderIdsPayload, + #[serde(rename = "ParentIndexNumber")] + pub season_number: Option, + #[serde(rename = "IndexNumber")] + pub episode_number: Option, + #[serde(rename = "Name")] + pub episode_name: Option, + pub series_name: Option, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + #[serde(rename_all = "PascalCase")] + pub struct EmbyWebhookPayload { + pub event: Option, + pub item: EmbyWebhookItemPayload, + pub series: Option, + pub playback_info: EmbyWebhookPlaybackInfoPayload, + } +} + +pub struct EmbyIntegration +{ + payload: String, + db: DatabaseConnection +} + +impl EmbyIntegration { + pub const fn new(payload: String, db: DatabaseConnection) -> Self { + Self { + payload, + db + } + } + async fn emby_progress( + &self + ) -> anyhow::Result<(Vec, Vec)> { + let payload: models::EmbyWebhookPayload = serde_json::from_str(&self.payload)?; + + let runtime = payload.item.run_time_ticks.ok_or_else(|| anyhow::anyhow!("No run time associated with this media"))?; + let position = payload.playback_info.position_ticks.ok_or_else(|| anyhow::anyhow!("No position associated with this media"))?; + + let (identifier, lot) = match payload.item.item_type.as_str() { + "Movie" => { + let id = payload.item.provider_ids.tmdb + .as_ref() + .ok_or_else(|| anyhow::anyhow!("No TMDb ID associated with this media"))?; + (id.clone(), MediaLot::Movie) + }, + "Episode" => { + let series_name = payload.item.series_name + .as_ref() + .ok_or_else(|| anyhow::anyhow!("No series name associated with this media"))?; + let episode_name = payload.item.episode_name + .as_ref() + .ok_or_else(|| anyhow::anyhow!("No episode name associated with this media"))?; + let db_show = self.get_show_by_episode_identifier(series_name, episode_name).await?; + (db_show.identifier, MediaLot::Show) + }, + _ => anyhow::bail!("Only movies and shows supported"), + }; + + Ok((vec![IntegrationMediaSeen { + identifier, + lot, + source: MediaSource::Tmdb, + progress: position / runtime * dec!(100), + show_season_number: payload.item.season_number, + show_episode_number: payload.item.episode_number, + provider_watched_on: Some("Emby".to_string()), + ..Default::default() + }], vec![])) + } +} + +#[async_trait] +impl ShowIdentifier for EmbyIntegration { + fn get_db(&self) -> &DatabaseConnection { + &self.db + } +} + +impl Integration for EmbyIntegration { + async fn progress(&self) -> anyhow::Result<(Vec, Vec)> { + self.emby_progress().await + } +} diff --git a/crates/services/integration/src/integration_type.rs b/crates/services/integration/src/integration_type.rs index afc05afa1c..0b94461786 100644 --- a/crates/services/integration/src/integration_type.rs +++ b/crates/services/integration/src/integration_type.rs @@ -3,5 +3,5 @@ use enums::MediaSource; pub enum IntegrationType { Komga(String, String, String, MediaSource), Jellyfin(String), - + Emby(String), } diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index 5378da2dd9..49f083f2ce 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -37,6 +37,7 @@ use crate::{ integration_type::IntegrationType, komga::KomgaIntegration }; +use crate::emby::EmbyIntegration; use crate::jellyfin::JellyfinIntegration; pub mod integration_type; @@ -44,6 +45,8 @@ mod integration; mod komga; mod jellyfin; +mod emby; +mod show_identifier; #[derive(Debug)] pub struct IntegrationService { @@ -201,105 +204,6 @@ impl IntegrationService { }) } - pub async fn emby_progress(&self, payload: &str) -> Result { - mod models { - use super::*; - - #[derive(Serialize, Deserialize, Debug, Clone)] - #[serde(rename_all = "PascalCase")] - pub struct EmbyWebhookPlaybackInfoPayload { - pub position_ticks: Option, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - #[serde(rename_all = "PascalCase")] - pub struct EmbyWebhookItemProviderIdsPayload { - pub tmdb: Option, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - #[serde(rename_all = "PascalCase")] - pub struct EmbyWebhookItemPayload { - pub run_time_ticks: Option, - #[serde(rename = "Type")] - pub item_type: String, - pub provider_ids: EmbyWebhookItemProviderIdsPayload, - #[serde(rename = "ParentIndexNumber")] - pub season_number: Option, - #[serde(rename = "IndexNumber")] - pub episode_number: Option, - #[serde(rename = "Name")] - pub episode_name: Option, - pub series_name: Option, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - #[serde(rename_all = "PascalCase")] - pub struct EmbyWebhookPayload { - pub event: Option, - pub item: EmbyWebhookItemPayload, - pub series: Option, - pub playback_info: EmbyWebhookPlaybackInfoPayload, - } - } - - let payload = serde_json::from_str::(payload)?; - - let identifier = if let Some(id) = payload.item.provider_ids.tmdb.as_ref() { - Some(id.clone()) - } else { - payload - .series - .as_ref() - .and_then(|s| s.provider_ids.tmdb.clone()) - }; - - if payload.item.run_time_ticks.is_none() { - bail!("No run time associated with this media") - } - if payload.playback_info.position_ticks.is_none() { - bail!("No position associated with this media") - } - - let runtime = payload.item.run_time_ticks.unwrap(); - let position = payload.playback_info.position_ticks.unwrap(); - - let (identifier, lot) = match payload.item.item_type.as_str() { - "Movie" => { - if identifier.is_none() { - bail!("No TMDb ID associated with this media") - } - - (identifier.unwrap().to_owned(), MediaLot::Movie) - } - "Episode" => { - if payload.item.episode_name.is_none() { - bail!("No episode name associated with this media") - } - - if payload.item.series_name.is_none() { - bail!("No series name associated with this media") - } - - let series_name = payload.item.series_name.unwrap(); - let episode_name = payload.item.episode_name.unwrap(); - let db_show = self - .get_show_by_episode_identifier(&series_name, &episode_name) - .await?; - (db_show.identifier, MediaLot::Show) - } - _ => bail!("Only movies and shows supported"), - }; - - Ok(IntegrationMediaSeen { - identifier, - lot, - source: MediaSource::Tmdb, - progress: position / runtime * dec!(100), - show_season_number: payload.item.season_number, - show_episode_number: payload.item.episode_number, - provider_watched_on: Some("Emby".to_string()), - ..Default::default() - }) - } - pub async fn kodi_progress(&self, payload: &str) -> Result { let mut payload = match serde_json::from_str::(payload) { Result::Ok(val) => val, @@ -515,6 +419,10 @@ impl IntegrationService { let jellyfin = JellyfinIntegration::new(payload); jellyfin.progress().await } + IntegrationType::Emby(payload) => { + let emby = EmbyIntegration::new(payload, self.db.clone()); + emby.progress().await + } } } } diff --git a/crates/services/integration/src/show_identifier.rs b/crates/services/integration/src/show_identifier.rs new file mode 100644 index 0000000000..54ad35135f --- /dev/null +++ b/crates/services/integration/src/show_identifier.rs @@ -0,0 +1,43 @@ +use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, ColumnTrait, Condition}; +use sea_query::{Expr, Func, Alias}; +use anyhow::{Result, bail}; +use sea_orm::prelude::async_trait::async_trait; +use sea_query::extension::postgres::PgExpr; +use database_utils::ilike_sql; +use crate::{metadata, MediaLot, MediaSource}; + +#[async_trait] +pub trait ShowIdentifier { + fn get_db(&self) -> &DatabaseConnection; + + async fn get_show_by_episode_identifier( + &self, + series: &str, + episode: &str, + ) -> Result { + let db_show = metadata::Entity::find() + .filter(metadata::Column::Lot.eq(MediaLot::Show)) + .filter(metadata::Column::Source.eq(MediaSource::Tmdb)) + .filter( + Condition::all() + .add( + Expr::expr(Func::cast_as( + Expr::col(metadata::Column::ShowSpecifics), + Alias::new("text"), + )) + .ilike(ilike_sql(episode)), + ) + .add(Expr::col(metadata::Column::Title).ilike(ilike_sql(series))), + ) + .one(self.get_db()) + .await?; + match db_show { + Some(show) => Ok(show), + None => bail!( + "No show found with Series {:#?} and Episode {:#?}", + series, + episode + ), + } + } +} diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index 27b0da62f8..8571be436e 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -4652,7 +4652,9 @@ impl MiscellaneousService { let service = self.get_integration_service(); let maybe_progress_update = match integration.provider { // IntegrationProvider::Kodi => service.kodi_progress(&payload).await, - // IntegrationProvider::Emby => service.emby_progress(&payload).await, + IntegrationProvider::Emby => service.process_progress(IntegrationType::Emby( + payload.clone() + )).await, IntegrationProvider::Jellyfin => service.process_progress(IntegrationType::Jellyfin( payload.clone() )).await, From 110d25c79dee5542dc2f1637a7ae8b8f1ad0cba8 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 15:22:54 -0700 Subject: [PATCH 51/61] Plex Integration Migration --- .../integration/src/integration_type.rs | 1 + crates/services/integration/src/lib.rs | 161 +----------------- crates/services/integration/src/plex.rs | 159 +++++++++++++++++ crates/services/miscellaneous/src/lib.rs | 13 +- 4 files changed, 175 insertions(+), 159 deletions(-) create mode 100644 crates/services/integration/src/plex.rs diff --git a/crates/services/integration/src/integration_type.rs b/crates/services/integration/src/integration_type.rs index 0b94461786..bf1c6becc3 100644 --- a/crates/services/integration/src/integration_type.rs +++ b/crates/services/integration/src/integration_type.rs @@ -4,4 +4,5 @@ pub enum IntegrationType { Komga(String, String, String, MediaSource), Jellyfin(String), Emby(String), + Plex(String, Option), } diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index 49f083f2ce..1ad86c5d8a 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -3,8 +3,7 @@ use std::future::Future; use anyhow::{anyhow, bail, Result}; use application_utils::get_base_http_client; use async_graphql::Result as GqlResult; -use database_models::{metadata, prelude::Metadata}; -use database_utils::ilike_sql; +use database_models::metadata; use enums::{MediaLot, MediaSource}; use media_models::{CommitMediaInput, IntegrationMediaCollection, IntegrationMediaSeen}; use providers::google_books::GoogleBooksService; @@ -15,13 +14,9 @@ use radarr_api_rs::{ }, models::{AddMovieOptions as RadarrAddMovieOptions, MovieResource as RadarrMovieResource}, }; -use regex::Regex; use reqwest::header::{HeaderValue, AUTHORIZATION}; -use rust_decimal::Decimal; use rust_decimal_macros::dec; -use sea_orm::{ColumnTrait, Condition, DatabaseConnection, EntityTrait, QueryFilter}; -use sea_query::{extension::postgres::PgExpr, Alias, Expr, Func}; -use serde::{Deserialize, Serialize}; +use sea_orm::DatabaseConnection; use sonarr_api_rs::{ apis::{ configuration::{ApiKey as SonarrApiKey, Configuration as SonarrConfiguration}, @@ -39,6 +34,7 @@ use crate::{ }; use crate::emby::EmbyIntegration; use crate::jellyfin::JellyfinIntegration; +use crate::plex::PlexIntegration; pub mod integration_type; mod integration; @@ -47,6 +43,7 @@ mod komga; mod jellyfin; mod emby; mod show_identifier; +mod plex; #[derive(Debug)] pub struct IntegrationService { @@ -58,152 +55,6 @@ impl IntegrationService { Self { db: db.clone() } } - // DEV: Fuzzy search for show by episode name and series name. - async fn get_show_by_episode_identifier( - &self, - series: &str, - episode: &str, - ) -> Result { - let db_show = Metadata::find() - .filter(metadata::Column::Lot.eq(MediaLot::Show)) - .filter(metadata::Column::Source.eq(MediaSource::Tmdb)) - .filter( - Condition::all() - .add( - Expr::expr(Func::cast_as( - Expr::col(metadata::Column::ShowSpecifics), - Alias::new("text"), - )) - .ilike(ilike_sql(episode)), - ) - .add(Expr::col(metadata::Column::Title).ilike(ilike_sql(series))), - ) - .one(&self.db) - .await?; - match db_show { - Some(show) => Ok(show), - None => bail!( - "No show found with Series {:#?} and Episode {:#?}", - series, - episode - ), - } - } - - - - pub async fn plex_progress( - &self, - payload: &str, - plex_user: Option, - ) -> Result { - mod models { - use super::*; - - #[derive(Serialize, Deserialize, Debug, Clone)] - pub struct PlexWebhookMetadataGuid { - pub id: String, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - pub struct PlexWebhookMetadataPayload { - #[serde(rename = "type")] - pub item_type: String, - #[serde(rename = "viewOffset")] - pub view_offset: Option, - pub duration: Decimal, - #[serde(rename = "grandparentTitle")] - pub show_name: Option, - #[serde(rename = "parentIndex")] - pub season_number: Option, - #[serde(rename = "index")] - pub episode_number: Option, - #[serde(rename = "Guid")] - pub guids: Vec, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - pub struct PlexWebhookAccount { - #[serde(rename = "title")] - pub plex_user: String, - } - #[derive(Serialize, Deserialize, Debug, Clone)] - pub struct PlexWebhookPayload { - #[serde(rename = "event")] - pub event_type: String, - pub user: bool, - pub owner: bool, - #[serde(rename = "Metadata")] - pub metadata: PlexWebhookMetadataPayload, - #[serde(rename = "Account")] - pub account: PlexWebhookAccount, - } - } - - tracing::debug!("Processing Plex payload {:#?}", payload); - - let payload_regex = Regex::new(r"\{.*\}").unwrap(); - let json_payload = payload_regex - .find(payload) - .map(|x| x.as_str()) - .unwrap_or(""); - let payload = match serde_json::from_str::(json_payload) { - Result::Ok(val) => val, - Result::Err(err) => bail!("Error during JSON payload deserialization {:#}", err), - }; - if let Some(plex_user) = plex_user { - if plex_user != payload.account.plex_user { - bail!( - "Ignoring non matching user {:#?}", - payload.account.plex_user - ); - } - } - match payload.event_type.as_str() { - "media.scrobble" | "media.play" | "media.pause" | "media.resume" | "media.stop" => {} - _ => bail!("Ignoring event type {:#?}", payload.event_type), - }; - - let tmdb_guid = payload - .metadata - .guids - .into_iter() - .find(|g| g.id.starts_with("tmdb://")); - - if tmdb_guid.is_none() { - bail!("No TMDb ID associated with this media") - } - let tmdb_guid = tmdb_guid.unwrap(); - let identifier = &tmdb_guid.id[7..]; - let (identifier, lot) = match payload.metadata.item_type.as_str() { - "movie" => (identifier.to_owned(), MediaLot::Movie), - "episode" => { - let series_name = payload.metadata.show_name.as_ref().unwrap(); - let db_show = self - .get_show_by_episode_identifier(series_name, identifier) - .await?; - (db_show.identifier, MediaLot::Show) - } - _ => bail!("Only movies and shows supported"), - }; - let progress = match payload.metadata.view_offset { - Some(offset) => offset / payload.metadata.duration * dec!(100), - None => match payload.event_type.as_str() { - "media.scrobble" => dec!(100), - _ => bail!("No position associated with this media"), - }, - }; - - Ok(IntegrationMediaSeen { - identifier, - lot, - source: MediaSource::Tmdb, - progress, - provider_watched_on: Some("Plex".to_string()), - show_season_number: payload.metadata.season_number, - show_episode_number: payload.metadata.episode_number, - ..Default::default() - }) - } - pub async fn kodi_progress(&self, payload: &str) -> Result { let mut payload = match serde_json::from_str::(payload) { Result::Ok(val) => val, @@ -423,6 +274,10 @@ impl IntegrationService { let emby = EmbyIntegration::new(payload, self.db.clone()); emby.progress().await } + IntegrationType::Plex(payload, plex_user) => { + let plex = PlexIntegration::new(payload, plex_user, self.db.clone()); + plex.progress().await + } } } } diff --git a/crates/services/integration/src/plex.rs b/crates/services/integration/src/plex.rs new file mode 100644 index 0000000000..daa1485411 --- /dev/null +++ b/crates/services/integration/src/plex.rs @@ -0,0 +1,159 @@ +use anyhow::{bail, Context}; +use async_graphql::async_trait::async_trait; +use regex::Regex; +use rust_decimal::Decimal; +use rust_decimal_macros::dec; +use sea_orm::DatabaseConnection; +use enums::{MediaLot, MediaSource}; +use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; +use crate::integration::Integration; +use crate::show_identifier::ShowIdentifier; + +mod models { + use rust_decimal::Decimal; + use serde::{Deserialize, Serialize}; + + #[derive(Serialize, Deserialize, Debug, Clone)] + pub struct PlexWebhookMetadataGuid { + pub id: String, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + pub struct PlexWebhookMetadataPayload { + #[serde(rename = "type")] + pub item_type: String, + #[serde(rename = "viewOffset")] + pub view_offset: Option, + pub duration: Decimal, + #[serde(rename = "grandparentTitle")] + pub show_name: Option, + #[serde(rename = "parentIndex")] + pub season_number: Option, + #[serde(rename = "index")] + pub episode_number: Option, + #[serde(rename = "Guid")] + pub guids: Vec, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + pub struct PlexWebhookAccount { + #[serde(rename = "title")] + pub plex_user: String, + } + #[derive(Serialize, Deserialize, Debug, Clone)] + pub struct PlexWebhookPayload { + #[serde(rename = "event")] + pub event_type: String, + pub user: bool, + pub owner: bool, + #[serde(rename = "Metadata")] + pub metadata: PlexWebhookMetadataPayload, + #[serde(rename = "Account")] + pub account: PlexWebhookAccount, + } +} + +pub struct PlexIntegration { + payload: String, + plex_user: Option, + db: DatabaseConnection, +} + +impl PlexIntegration { + pub const fn new(payload: String, plex_user: Option, db: DatabaseConnection) -> Self { + Self { + payload, + plex_user, + db + } + } + + fn parse_payload(payload: &str) -> anyhow::Result { + let payload_regex = Regex::new(r"\{.*\}").unwrap(); + let json_payload = payload_regex + .find(payload) + .map(|x| x.as_str()) + .unwrap_or(""); + serde_json::from_str(json_payload).context("Error during JSON payload deserialization") + } + + fn get_tmdb_identifier(guids: &[models::PlexWebhookMetadataGuid]) -> anyhow::Result<&str> { + guids + .iter() + .find(|g| g.id.starts_with("tmdb://")) + .map(|g| &g.id[7..]) + .ok_or_else(|| anyhow::anyhow!("No TMDb ID associated with this media")) + } + + async fn get_media_info<'a>( + &self, + metadata: &'a models::PlexWebhookMetadataPayload, + identifier: &'a str, + ) -> anyhow::Result<(String, MediaLot)> { + match metadata.item_type.as_str() { + "movie" => Ok((identifier.to_owned(), MediaLot::Movie)), + "episode" => { + let series_name = metadata.show_name.as_ref().context("Show name missing")?; + let db_show = self.get_show_by_episode_identifier(series_name, identifier).await?; + Ok((db_show.identifier, MediaLot::Show)) + } + _ => bail!("Only movies and shows supported"), + } + } + + fn calculate_progress(payload: &models::PlexWebhookPayload) -> anyhow::Result { + match payload.metadata.view_offset { + Some(offset) => Ok(offset / payload.metadata.duration * dec!(100)), + None if payload.event_type == "media.scrobble" => Ok(dec!(100)), + None => bail!("No position associated with this media"), + } + } + + async fn plex_progress( + &self + ) -> anyhow::Result<(Vec, Vec)> { + tracing::debug!("Processing Plex payload {:#?}", self.payload); + + let payload = Self::parse_payload(&self.payload)?; + + if let Some(plex_user) = &self.plex_user { + if *plex_user != payload.account.plex_user { + bail!( + "Ignoring non matching user {:#?}", + payload.account.plex_user + ); + } + } + + match payload.event_type.as_str() { + "media.scrobble" | "media.play" | "media.pause" | "media.resume" | "media.stop" => {} + _ => bail!("Ignoring event type {:#?}", payload.event_type), + }; + + let identifier = Self::get_tmdb_identifier(&payload.metadata.guids)?; + let (identifier, lot) = self.get_media_info(&payload.metadata, identifier).await?; + let progress = Self::calculate_progress(&payload)?; + + Ok((vec![IntegrationMediaSeen { + identifier, + lot, + source: MediaSource::Tmdb, + progress, + provider_watched_on: Some("Plex".to_string()), + show_season_number: payload.metadata.season_number, + show_episode_number: payload.metadata.episode_number, + ..Default::default() + }], vec![])) + } +} + +#[async_trait] +impl ShowIdentifier for PlexIntegration { + fn get_db(&self) -> &DatabaseConnection { + &self.db + } +} + +impl Integration for PlexIntegration { + async fn progress(&self) -> anyhow::Result<(Vec, Vec)> { + self.plex_progress().await + } +} diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index 8571be436e..01c142466a 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -4658,12 +4658,13 @@ impl MiscellaneousService { IntegrationProvider::Jellyfin => service.process_progress(IntegrationType::Jellyfin( payload.clone() )).await, - // IntegrationProvider::Plex => { - // let specifics = integration.clone().provider_specifics.unwrap(); - // service - // .plex_progress(&payload, specifics.plex_username) - // .await - // } + IntegrationProvider::Plex => { + let specifics = integration.clone().provider_specifics.unwrap(); + service.process_progress(IntegrationType::Plex( + payload.clone(), + specifics.plex_username + )).await + }, _ => return Err(Error::new("Unsupported integration source".to_owned())), }; match maybe_progress_update { From bc5dfcc0e226624251e38e700f7727f73cd880bf Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 16:56:29 -0700 Subject: [PATCH 52/61] Audiobookshelf Integration Migration (partial) --- .../integration/src/audiobookshelf.rs | 164 ++++++++++++++++++ .../integration/src/integration_type.rs | 2 + crates/services/integration/src/lib.rs | 152 +--------------- crates/services/miscellaneous/src/lib.rs | 13 +- 4 files changed, 180 insertions(+), 151 deletions(-) create mode 100644 crates/services/integration/src/audiobookshelf.rs diff --git a/crates/services/integration/src/audiobookshelf.rs b/crates/services/integration/src/audiobookshelf.rs new file mode 100644 index 0000000000..da8e0d392c --- /dev/null +++ b/crates/services/integration/src/audiobookshelf.rs @@ -0,0 +1,164 @@ +use anyhow::anyhow; +use media_models::{IntegrationMediaSeen, IntegrationMediaCollection}; +use providers::google_books::GoogleBooksService; +use reqwest::header::{AUTHORIZATION, HeaderValue}; +use rust_decimal_macros::dec; +use application_utils::get_base_http_client; +use enums::{MediaLot, MediaSource}; +use crate::integration::Integration; +use specific_models::audiobookshelf; + +pub struct AudiobookshelfIntegration { + base_url: String, + access_token: String, + isbn_service: GoogleBooksService +} + +impl AudiobookshelfIntegration { + pub fn new( + base_url: String, + access_token: String, + isbn_service: GoogleBooksService + ) -> Self + { + Self { + base_url, + access_token, + isbn_service + } + } +} + +impl Integration for AudiobookshelfIntegration { + async fn progress(&self) -> anyhow::Result<(Vec, Vec)> { + let client = get_base_http_client( + &format!("{}/api/", self.base_url), + Some(vec![( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", self.access_token)).unwrap(), + )]), + ); + + let resp = client + .get("me/items-in-progress") + .send() + .await + .map_err(|e| anyhow!(e))? + .json::() + .await + .map_err(|e| anyhow!(e))?; + + tracing::debug!("Got response for items in progress {:?}", resp); + + let mut media_items = vec![]; + + for item in resp.library_items.iter() { + let metadata = item.media.clone().unwrap().metadata; + let (progress_id, identifier, lot, source, podcast_episode_number) = + if Some("epub".to_string()) == item.media.as_ref().unwrap().ebook_format { + match &metadata.isbn { + Some(isbn) => match self.isbn_service.id_from_isbn(isbn).await { + Some(id) => ( + item.id.clone(), + id, + MediaLot::Book, + MediaSource::GoogleBooks, + None, + ), + _ => { + tracing::debug!("No Google Books ID found for ISBN {:#?}", isbn); + continue; + } + }, + _ => { + tracing::debug!("No ISBN found for item {:#?}", item); + continue; + } + } + } else if let Some(asin) = metadata.asin.clone() { + ( + item.id.clone(), + asin, + MediaLot::AudioBook, + MediaSource::Audible, + None, + ) + } + // else if let Some(itunes_id) = metadata.itunes_id.clone() { + // match &item.recent_episode { + // Some(pe) => { + // let lot = MediaLot::Podcast; + // let source = MediaSource::Itunes; + // let podcast = (self.commit_metadata)(CommitMediaInput { + // identifier: itunes_id.clone(), + // lot, + // source, + // force_update: None, + // }).await.map_err(|e| anyhow!("Failed to commit metadata: {:?}", e))?; + // match podcast + // .podcast_specifics + // .and_then(|p| p.episode_by_name(&pe.title)) + // { + // Some(episode) => ( + // format!("{}/{}", item.id, pe.id), + // itunes_id, + // lot, + // source, + // Some(episode), + // ), + // _ => { + // tracing::debug!( + // "No podcast found for iTunes ID {:#?}", + // itunes_id + // ); + // continue; + // } + // } + // } + // _ => { + // tracing::debug!("No recent episode found for item {:#?}", item); + // continue; + // } + // } + // } + else { + tracing::debug!("No ASIN, ISBN or iTunes ID found for item {:#?}", item); + continue; + }; + + match client + .get(format!("me/progress/{}", progress_id)) + .send() + .await + .map_err(|e| anyhow!(e))? + .json::() + .await + .map_err(|e| anyhow!(e)) + { + Ok(resp) => { + tracing::debug!("Got response for individual item progress {:?}", resp); + let progress = if let Some(ebook_progress) = resp.ebook_progress { + ebook_progress + } else { + resp.progress + }; + media_items.push(IntegrationMediaSeen { + lot, + source, + identifier, + podcast_episode_number, + progress: progress * dec!(100), + provider_watched_on: Some("Audiobookshelf".to_string()), + ..Default::default() + }); + } + Err(e) => { + tracing::debug!("Error getting progress for item {:?}: {:?}", item, e); + continue; + } + }; + } + + Ok((media_items, vec![])) + } +} diff --git a/crates/services/integration/src/integration_type.rs b/crates/services/integration/src/integration_type.rs index bf1c6becc3..a821606570 100644 --- a/crates/services/integration/src/integration_type.rs +++ b/crates/services/integration/src/integration_type.rs @@ -1,8 +1,10 @@ use enums::MediaSource; +use providers::google_books::GoogleBooksService; pub enum IntegrationType { Komga(String, String, String, MediaSource), Jellyfin(String), Emby(String), Plex(String, Option), + Audiobookshelf(String, String, GoogleBooksService) } diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index 1ad86c5d8a..5edf80789d 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -1,12 +1,7 @@ -use std::future::Future; - -use anyhow::{anyhow, bail, Result}; -use application_utils::get_base_http_client; -use async_graphql::Result as GqlResult; +use anyhow::{bail, Result}; use database_models::metadata; use enums::{MediaLot, MediaSource}; -use media_models::{CommitMediaInput, IntegrationMediaCollection, IntegrationMediaSeen}; -use providers::google_books::GoogleBooksService; +use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; use radarr_api_rs::{ apis::{ configuration::{ApiKey as RadarrApiKey, Configuration as RadarrConfiguration}, @@ -14,8 +9,6 @@ use radarr_api_rs::{ }, models::{AddMovieOptions as RadarrAddMovieOptions, MovieResource as RadarrMovieResource}, }; -use reqwest::header::{HeaderValue, AUTHORIZATION}; -use rust_decimal_macros::dec; use sea_orm::DatabaseConnection; use sonarr_api_rs::{ apis::{ @@ -24,7 +17,6 @@ use sonarr_api_rs::{ }, models::{AddSeriesOptions as SonarrAddSeriesOptions, SeriesResource as SonarrSeriesResource}, }; -use specific_models::audiobookshelf as audiobookshelf_models; use traits::TraceOk; use crate::{ @@ -32,6 +24,7 @@ use crate::{ integration_type::IntegrationType, komga::KomgaIntegration }; +use crate::audiobookshelf::AudiobookshelfIntegration; use crate::emby::EmbyIntegration; use crate::jellyfin::JellyfinIntegration; use crate::plex::PlexIntegration; @@ -44,6 +37,7 @@ mod jellyfin; mod emby; mod show_identifier; mod plex; +mod audiobookshelf; #[derive(Debug)] pub struct IntegrationService { @@ -65,139 +59,7 @@ impl IntegrationService { Ok(payload) } - pub async fn audiobookshelf_progress( - &self, - base_url: &str, - access_token: &str, - isbn_service: &GoogleBooksService, - commit_metadata: impl Fn(CommitMediaInput) -> F, - ) -> Result<(Vec, Vec)> - where - F: Future>, - { - let client = get_base_http_client( - &format!("{}/api/", base_url), - Some(vec![( - AUTHORIZATION, - HeaderValue::from_str(&format!("Bearer {access_token}")).unwrap(), - )]), - ); - let resp = client - .get("me/items-in-progress") - .send() - .await - .map_err(|e| anyhow!(e))? - .json::() - .await - .unwrap(); - tracing::debug!("Got response for items in progress {:?}", resp); - let mut media_items = vec![]; - for item in resp.library_items.iter() { - let metadata = item.media.clone().unwrap().metadata; - let (progress_id, identifier, lot, source, podcast_episode_number) = - if Some("epub".to_string()) == item.media.as_ref().unwrap().ebook_format { - match &metadata.isbn { - Some(isbn) => match isbn_service.id_from_isbn(isbn).await { - Some(id) => ( - item.id.clone(), - id, - MediaLot::Book, - MediaSource::GoogleBooks, - None, - ), - _ => { - tracing::debug!("No Google Books ID found for ISBN {:#?}", isbn); - continue; - } - }, - _ => { - tracing::debug!("No ISBN found for item {:#?}", item); - continue; - } - } - } else if let Some(asin) = metadata.asin.clone() { - ( - item.id.clone(), - asin, - MediaLot::AudioBook, - MediaSource::Audible, - None, - ) - } else if let Some(itunes_id) = metadata.itunes_id.clone() { - match &item.recent_episode { - Some(pe) => { - let lot = MediaLot::Podcast; - let source = MediaSource::Itunes; - let podcast = commit_metadata(CommitMediaInput { - identifier: itunes_id.clone(), - lot, - source, - ..Default::default() - }) - .await - .unwrap(); - match podcast - .podcast_specifics - .and_then(|p| p.episode_by_name(&pe.title)) - { - Some(episode) => ( - format!("{}/{}", item.id, pe.id), - itunes_id, - lot, - source, - Some(episode), - ), - _ => { - tracing::debug!( - "No podcast found for iTunes ID {:#?}", - itunes_id - ); - continue; - } - } - } - _ => { - tracing::debug!("No recent episode found for item {:#?}", item); - continue; - } - } - } else { - tracing::debug!("No ASIN, ISBN or iTunes ID found for item {:#?}", item); - continue; - }; - match client - .get(format!("me/progress/{}", progress_id)) - .send() - .await - .map_err(|e| anyhow!(e))? - .json::() - .await - { - Ok(resp) => { - tracing::debug!("Got response for individual item progress {:?}", resp); - let progress = if let Some(ebook_progress) = resp.ebook_progress { - ebook_progress - } else { - resp.progress - }; - media_items.push(IntegrationMediaSeen { - lot, - source, - identifier, - podcast_episode_number, - progress: progress * dec!(100), - provider_watched_on: Some("Audiobookshelf".to_string()), - ..Default::default() - }); - } - _ => { - tracing::debug!("No progress found for item {:?}", item); - continue; - } - }; - } - Ok((media_items, vec![])) - } + pub async fn radarr_push( &self, @@ -278,6 +140,10 @@ impl IntegrationService { let plex = PlexIntegration::new(payload, plex_user, self.db.clone()); plex.progress().await } + IntegrationType::Audiobookshelf(base_url, access_token, isbn_service) => { + let audiobookshelf = AudiobookshelfIntegration::new(base_url, access_token, isbn_service); + audiobookshelf.progress().await + } } } } diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index 01c142466a..86295e2fa1 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -4433,14 +4433,11 @@ impl MiscellaneousService { let response = match integration.provider { IntegrationProvider::Audiobookshelf => { let specifics = integration.clone().provider_specifics.unwrap(); - integration_service - .audiobookshelf_progress( - &specifics.audiobookshelf_base_url.unwrap(), - &specifics.audiobookshelf_token.unwrap(), - &self.get_isbn_service().await.unwrap(), - |input| self.commit_metadata(input), - ) - .await + integration_service.process_progress(IntegrationType::Audiobookshelf( + specifics.audiobookshelf_base_url.unwrap(), + specifics.audiobookshelf_token.unwrap(), + self.get_isbn_service().await.unwrap(), + )).await } IntegrationProvider::Komga => { let specifics = integration.clone().provider_specifics.unwrap(); From 45db825e7d5e30451056e7fb89c2b6a8f4671f1e Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 17:06:00 -0700 Subject: [PATCH 53/61] Kodi Integration Migration --- .../integration/src/integration_type.rs | 3 +- crates/services/integration/src/kodi.rs | 33 +++++++++++++++++++ crates/services/integration/src/lib.rs | 20 ++++------- crates/services/miscellaneous/src/lib.rs | 4 ++- 4 files changed, 45 insertions(+), 15 deletions(-) create mode 100644 crates/services/integration/src/kodi.rs diff --git a/crates/services/integration/src/integration_type.rs b/crates/services/integration/src/integration_type.rs index a821606570..b6362337b5 100644 --- a/crates/services/integration/src/integration_type.rs +++ b/crates/services/integration/src/integration_type.rs @@ -6,5 +6,6 @@ pub enum IntegrationType { Jellyfin(String), Emby(String), Plex(String, Option), - Audiobookshelf(String, String, GoogleBooksService) + Audiobookshelf(String, String, GoogleBooksService), + Kodi(String) } diff --git a/crates/services/integration/src/kodi.rs b/crates/services/integration/src/kodi.rs new file mode 100644 index 0000000000..8077a11912 --- /dev/null +++ b/crates/services/integration/src/kodi.rs @@ -0,0 +1,33 @@ +use anyhow::bail; +use enums::MediaSource; +use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; +use crate::integration::Integration; + +pub struct KodiIntegration { + payload: String +} +impl KodiIntegration { + pub const fn new(payload: String) -> Self { + Self { + payload + } + } + + async fn kodi_progress( + &self + ) -> anyhow::Result<(Vec, Vec)> { + let mut payload = match serde_json::from_str::(&self.payload) { + Ok(val) => val, + Err(err) => bail!(err), + }; + payload.source = MediaSource::Tmdb; + payload.provider_watched_on = Some("Kodi".to_string()); + Ok((vec![payload], vec![])) + } +} + +impl Integration for KodiIntegration { + async fn progress(&self) -> anyhow::Result<(Vec, Vec)> { + self.kodi_progress().await + } +} diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index 5edf80789d..45a9616b28 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -1,4 +1,4 @@ -use anyhow::{bail, Result}; +use anyhow::Result; use database_models::metadata; use enums::{MediaLot, MediaSource}; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; @@ -27,6 +27,7 @@ use crate::{ use crate::audiobookshelf::AudiobookshelfIntegration; use crate::emby::EmbyIntegration; use crate::jellyfin::JellyfinIntegration; +use crate::kodi::KodiIntegration; use crate::plex::PlexIntegration; pub mod integration_type; @@ -38,6 +39,7 @@ mod emby; mod show_identifier; mod plex; mod audiobookshelf; +mod kodi; #[derive(Debug)] pub struct IntegrationService { @@ -49,18 +51,6 @@ impl IntegrationService { Self { db: db.clone() } } - pub async fn kodi_progress(&self, payload: &str) -> Result { - let mut payload = match serde_json::from_str::(payload) { - Result::Ok(val) => val, - Result::Err(err) => bail!(err), - }; - payload.source = MediaSource::Tmdb; - payload.provider_watched_on = Some("Kodi".to_string()); - Ok(payload) - } - - - pub async fn radarr_push( &self, radarr_base_url: String, @@ -144,6 +134,10 @@ impl IntegrationService { let audiobookshelf = AudiobookshelfIntegration::new(base_url, access_token, isbn_service); audiobookshelf.progress().await } + IntegrationType::Kodi(payload) => { + let kodi = KodiIntegration::new(payload); + kodi.progress().await + } } } } diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index 86295e2fa1..edfd889451 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -4648,7 +4648,9 @@ impl MiscellaneousService { } let service = self.get_integration_service(); let maybe_progress_update = match integration.provider { - // IntegrationProvider::Kodi => service.kodi_progress(&payload).await, + IntegrationProvider::Kodi => service.process_progress(IntegrationType::Kodi( + payload.clone() + )).await, IntegrationProvider::Emby => service.process_progress(IntegrationType::Emby( payload.clone() )).await, From 5f131ed190a9e34e4b188e326beef63ffd95592b Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 17:42:18 -0700 Subject: [PATCH 54/61] Sonarr Integration Migration --- .../services/integration/src/integration.rs | 4 ++ .../integration/src/integration_type.rs | 3 +- crates/services/integration/src/lib.rs | 61 +++++++---------- crates/services/integration/src/sonarr.rs | 67 +++++++++++++++++++ crates/services/miscellaneous/src/lib.rs | 16 ++--- 5 files changed, 104 insertions(+), 47 deletions(-) create mode 100644 crates/services/integration/src/sonarr.rs diff --git a/crates/services/integration/src/integration.rs b/crates/services/integration/src/integration.rs index eb313da065..c4b4208c3f 100644 --- a/crates/services/integration/src/integration.rs +++ b/crates/services/integration/src/integration.rs @@ -6,3 +6,7 @@ use crate::IntegrationMediaSeen; pub trait Integration { async fn progress(&self) -> Result<(Vec, Vec)>; } + +pub trait PushIntegration { + async fn push(&self) -> Result<()>; +} diff --git a/crates/services/integration/src/integration_type.rs b/crates/services/integration/src/integration_type.rs index b6362337b5..d30928568d 100644 --- a/crates/services/integration/src/integration_type.rs +++ b/crates/services/integration/src/integration_type.rs @@ -7,5 +7,6 @@ pub enum IntegrationType { Emby(String), Plex(String, Option), Audiobookshelf(String, String, GoogleBooksService), - Kodi(String) + Kodi(String), + Sonarr(String, String, i32, String, String) } diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index 45a9616b28..4048ccf072 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -10,13 +10,7 @@ use radarr_api_rs::{ models::{AddMovieOptions as RadarrAddMovieOptions, MovieResource as RadarrMovieResource}, }; use sea_orm::DatabaseConnection; -use sonarr_api_rs::{ - apis::{ - configuration::{ApiKey as SonarrApiKey, Configuration as SonarrConfiguration}, - series_api::api_v3_series_post as sonarr_api_v3_series_post, - }, - models::{AddSeriesOptions as SonarrAddSeriesOptions, SeriesResource as SonarrSeriesResource}, -}; + use traits::TraceOk; use crate::{ @@ -26,9 +20,11 @@ use crate::{ }; use crate::audiobookshelf::AudiobookshelfIntegration; use crate::emby::EmbyIntegration; +use crate::integration::PushIntegration; use crate::jellyfin::JellyfinIntegration; use crate::kodi::KodiIntegration; use crate::plex::PlexIntegration; +use crate::sonarr::SonarrIntegration; pub mod integration_type; mod integration; @@ -40,6 +36,7 @@ mod show_identifier; mod plex; mod audiobookshelf; mod kodi; +mod sonarr; #[derive(Debug)] pub struct IntegrationService { @@ -79,36 +76,25 @@ impl IntegrationService { .trace_ok(); Ok(()) } - - pub async fn sonarr_push( - &self, - sonarr_base_url: String, - sonarr_api_key: String, - sonarr_profile_id: i32, - sonarr_root_folder_path: String, - tvdb_id: String, - ) -> Result<()> { - let mut configuration = SonarrConfiguration::new(); - configuration.base_path = sonarr_base_url; - configuration.api_key = Some(SonarrApiKey { - key: sonarr_api_key, - prefix: None, - }); - let mut resource = SonarrSeriesResource::new(); - resource.title = Some(Some(tvdb_id.clone())); - resource.tvdb_id = Some(tvdb_id.parse().unwrap()); - resource.quality_profile_id = Some(sonarr_profile_id); - resource.root_folder_path = Some(Some(sonarr_root_folder_path.clone())); - resource.monitored = Some(true); - resource.season_folder = Some(true); - let mut options = SonarrAddSeriesOptions::new(); - options.search_for_missing_episodes = Some(true); - resource.add_options = Some(Box::new(options)); - tracing::debug!("Pushing series to Sonarr {:?}", resource); - sonarr_api_v3_series_post(&configuration, Some(resource)) - .await - .trace_ok(); - Ok(()) + pub async fn push(&self, integration_type: IntegrationType) -> Result<()> { + match integration_type { + IntegrationType::Sonarr( + sonarr_base_url, + sonarr_api_key, + sonarr_profile_id, + sonarr_root_folder_path, + tvdb_id) => { + let sonarr = SonarrIntegration::new( + sonarr_base_url, + sonarr_api_key, + sonarr_profile_id, + sonarr_root_folder_path, + tvdb_id + ); + sonarr.push().await + } + _ => Ok(()) + } } pub async fn process_progress(&self, integration_type: IntegrationType) @@ -138,6 +124,7 @@ impl IntegrationService { let kodi = KodiIntegration::new(payload); kodi.progress().await } + _ => Ok((vec![],vec![])) } } } diff --git a/crates/services/integration/src/sonarr.rs b/crates/services/integration/src/sonarr.rs new file mode 100644 index 0000000000..90c8819ad0 --- /dev/null +++ b/crates/services/integration/src/sonarr.rs @@ -0,0 +1,67 @@ +use crate::integration::PushIntegration; +use sonarr_api_rs::{ + apis::{ + configuration::{ApiKey as SonarrApiKey, Configuration as SonarrConfiguration}, + series_api::api_v3_series_post as sonarr_api_v3_series_post, + }, + models::{AddSeriesOptions as SonarrAddSeriesOptions, SeriesResource as SonarrSeriesResource}, +}; +use traits::TraceOk; + +pub struct SonarrIntegration { + sonarr_base_url: String, + sonarr_api_key: String, + sonarr_profile_id: i32, + sonarr_root_folder_path: String, + tvdb_id: String, +} + +impl SonarrIntegration { + pub const fn new( + sonarr_base_url: String, + sonarr_api_key: String, + sonarr_profile_id: i32, + sonarr_root_folder_path: String, + tvdb_id: String, + ) -> Self { + Self { + sonarr_base_url, + sonarr_api_key, + sonarr_profile_id, + sonarr_root_folder_path, + tvdb_id + } + } + + async fn sonarr_push( + &self + ) -> anyhow::Result<()> { + let mut configuration = SonarrConfiguration::new(); + configuration.base_path = self.sonarr_base_url.clone(); + configuration.api_key = Some(SonarrApiKey { + key: self.sonarr_api_key.clone(), + prefix: None, + }); + let mut resource = SonarrSeriesResource::new(); + resource.title = Some(Some(self.tvdb_id.clone())); + resource.tvdb_id = Some(self.tvdb_id.parse().unwrap()); + resource.quality_profile_id = Some(self.sonarr_profile_id); + resource.root_folder_path = Some(Some(self.sonarr_root_folder_path.clone())); + resource.monitored = Some(true); + resource.season_folder = Some(true); + let mut options = SonarrAddSeriesOptions::new(); + options.search_for_missing_episodes = Some(true); + resource.add_options = Some(Box::new(options)); + tracing::debug!("Pushing series to Sonarr {:?}", resource); + sonarr_api_v3_series_post(&configuration, Some(resource)) + .await + .trace_ok(); + Ok(()) + } +} + +impl PushIntegration for SonarrIntegration { + async fn push(&self) -> anyhow::Result<()> { + self.sonarr_push().await + } +} diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index edfd889451..bda3adedbf 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -4571,15 +4571,13 @@ impl MiscellaneousService { .await } IntegrationProvider::Sonarr => { - integration_service - .sonarr_push( - specifics.sonarr_base_url.unwrap(), - specifics.sonarr_api_key.unwrap(), - specifics.sonarr_profile_id.unwrap(), - specifics.sonarr_root_folder_path.unwrap(), - entity_id, - ) - .await + integration_service.push(IntegrationType::Sonarr( + specifics.sonarr_base_url.unwrap(), + specifics.sonarr_api_key.unwrap(), + specifics.sonarr_profile_id.unwrap(), + specifics.sonarr_root_folder_path.unwrap(), + entity_id, + )).await } _ => unreachable!(), }; From be9c60897165d575984fa681c2915f43f69a331b Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 17:50:38 -0700 Subject: [PATCH 55/61] Radarr Integration Migration --- .../integration/src/integration_type.rs | 3 +- crates/services/integration/src/lib.rs | 55 +++++----------- crates/services/integration/src/radarr.rs | 65 +++++++++++++++++++ crates/services/miscellaneous/src/lib.rs | 16 ++--- 4 files changed, 91 insertions(+), 48 deletions(-) create mode 100644 crates/services/integration/src/radarr.rs diff --git a/crates/services/integration/src/integration_type.rs b/crates/services/integration/src/integration_type.rs index d30928568d..d5135e6f24 100644 --- a/crates/services/integration/src/integration_type.rs +++ b/crates/services/integration/src/integration_type.rs @@ -8,5 +8,6 @@ pub enum IntegrationType { Plex(String, Option), Audiobookshelf(String, String, GoogleBooksService), Kodi(String), - Sonarr(String, String, i32, String, String) + Sonarr(String, String, i32, String, String), + Radarr(String, String, i32, String, String) } diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index 4048ccf072..f0a209b781 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -2,17 +2,8 @@ use anyhow::Result; use database_models::metadata; use enums::{MediaLot, MediaSource}; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; -use radarr_api_rs::{ - apis::{ - configuration::{ApiKey as RadarrApiKey, Configuration as RadarrConfiguration}, - movie_api::api_v3_movie_post as radarr_api_v3_movie_post, - }, - models::{AddMovieOptions as RadarrAddMovieOptions, MovieResource as RadarrMovieResource}, -}; use sea_orm::DatabaseConnection; -use traits::TraceOk; - use crate::{ integration::Integration, integration_type::IntegrationType, @@ -24,6 +15,7 @@ use crate::integration::PushIntegration; use crate::jellyfin::JellyfinIntegration; use crate::kodi::KodiIntegration; use crate::plex::PlexIntegration; +use crate::radarr::RadarrIntegration; use crate::sonarr::SonarrIntegration; pub mod integration_type; @@ -37,6 +29,7 @@ mod plex; mod audiobookshelf; mod kodi; mod sonarr; +mod radarr; #[derive(Debug)] pub struct IntegrationService { @@ -47,35 +40,6 @@ impl IntegrationService { pub fn new(db: &DatabaseConnection) -> Self { Self { db: db.clone() } } - - pub async fn radarr_push( - &self, - radarr_base_url: String, - radarr_api_key: String, - radarr_profile_id: i32, - radarr_root_folder_path: String, - tmdb_id: String, - ) -> Result<()> { - let mut configuration = RadarrConfiguration::new(); - configuration.base_path = radarr_base_url; - configuration.api_key = Some(RadarrApiKey { - key: radarr_api_key, - prefix: None, - }); - let mut resource = RadarrMovieResource::new(); - resource.tmdb_id = Some(tmdb_id.parse().unwrap()); - resource.quality_profile_id = Some(radarr_profile_id); - resource.root_folder_path = Some(Some(radarr_root_folder_path.clone())); - resource.monitored = Some(true); - let mut options = RadarrAddMovieOptions::new(); - options.search_for_movie = Some(true); - resource.add_options = Some(Box::new(options)); - tracing::debug!("Pushing movie to Radarr {:?}", resource); - radarr_api_v3_movie_post(&configuration, Some(resource)) - .await - .trace_ok(); - Ok(()) - } pub async fn push(&self, integration_type: IntegrationType) -> Result<()> { match integration_type { IntegrationType::Sonarr( @@ -93,6 +57,21 @@ impl IntegrationService { ); sonarr.push().await } + IntegrationType::Radarr( + radarr_base_url, + radarr_api_key, + radarr_profile_id, + radarr_root_folder_path, + tmdb_id) => { + let radarr = RadarrIntegration::new( + radarr_base_url, + radarr_api_key, + radarr_profile_id, + radarr_root_folder_path, + tmdb_id + ); + radarr.push().await + } _ => Ok(()) } } diff --git a/crates/services/integration/src/radarr.rs b/crates/services/integration/src/radarr.rs new file mode 100644 index 0000000000..d450555003 --- /dev/null +++ b/crates/services/integration/src/radarr.rs @@ -0,0 +1,65 @@ +use crate::integration::PushIntegration; +use radarr_api_rs::{ + apis::{ + configuration::{ApiKey as RadarrApiKey, Configuration as RadarrConfiguration}, + movie_api::api_v3_movie_post as radarr_api_v3_movie_post, + }, + models::{AddMovieOptions as RadarrAddMovieOptions, MovieResource as RadarrMovieResource}, +}; +use traits::TraceOk; + +pub struct RadarrIntegration { + radarr_base_url: String, + radarr_api_key: String, + radarr_profile_id: i32, + radarr_root_folder_path: String, + tmdb_id: String, +} + +impl RadarrIntegration { + pub const fn new( + radarr_base_url: String, + radarr_api_key: String, + radarr_profile_id: i32, + radarr_root_folder_path: String, + tmdb_id: String, + ) -> Self { + Self { + radarr_base_url, + radarr_api_key, + radarr_profile_id, + radarr_root_folder_path, + tmdb_id + } + } + + pub async fn radarr_push( + &self + ) -> anyhow::Result<()> { + let mut configuration = RadarrConfiguration::new(); + configuration.base_path = self.radarr_base_url.clone(); + configuration.api_key = Some(RadarrApiKey { + key: self.radarr_api_key.clone(), + prefix: None, + }); + let mut resource = RadarrMovieResource::new(); + resource.tmdb_id = Some(self.tmdb_id.parse().unwrap()); + resource.quality_profile_id = Some(self.radarr_profile_id); + resource.root_folder_path = Some(Some(self.radarr_root_folder_path.clone())); + resource.monitored = Some(true); + let mut options = RadarrAddMovieOptions::new(); + options.search_for_movie = Some(true); + resource.add_options = Some(Box::new(options)); + tracing::debug!("Pushing movie to Radarr {:?}", resource); + radarr_api_v3_movie_post(&configuration, Some(resource)) + .await + .trace_ok(); + Ok(()) + } +} + +impl PushIntegration for RadarrIntegration { + async fn push(&self) -> anyhow::Result<()> { + self.radarr_push().await + } +} diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index bda3adedbf..e8e7f37911 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -4560,15 +4560,13 @@ impl MiscellaneousService { if let Some(entity_id) = maybe_entity_id { let _push_result = match integration.provider { IntegrationProvider::Radarr => { - integration_service - .radarr_push( - specifics.radarr_base_url.unwrap(), - specifics.radarr_api_key.unwrap(), - specifics.radarr_profile_id.unwrap(), - specifics.radarr_root_folder_path.unwrap(), - entity_id, - ) - .await + integration_service.push(IntegrationType::Radarr( + specifics.radarr_base_url.unwrap(), + specifics.radarr_api_key.unwrap(), + specifics.radarr_profile_id.unwrap(), + specifics.radarr_root_folder_path.unwrap(), + entity_id, + )).await } IntegrationProvider::Sonarr => { integration_service.push(IntegrationType::Sonarr( From 3fc61c016a25a23e3d730c7541e8d9e410c433d8 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 19:56:44 -0700 Subject: [PATCH 56/61] Ran formatter --- .../integration/src/audiobookshelf.rs | 93 +++++---- crates/services/integration/src/emby.rs | 93 +++++---- .../services/integration/src/integration.rs | 4 +- .../integration/src/integration_type.rs | 2 +- crates/services/integration/src/jellyfin.rs | 58 ++++-- crates/services/integration/src/kodi.rs | 14 +- crates/services/integration/src/komga.rs | 52 ++--- crates/services/integration/src/lib.rs | 59 +++--- crates/services/integration/src/plex.rs | 40 ++-- crates/services/integration/src/radarr.rs | 10 +- .../integration/src/show_identifier.rs | 17 +- crates/services/integration/src/sonarr.rs | 10 +- crates/services/miscellaneous/src/lib.rs | 179 ++++++++++-------- 13 files changed, 341 insertions(+), 290 deletions(-) diff --git a/crates/services/integration/src/audiobookshelf.rs b/crates/services/integration/src/audiobookshelf.rs index da8e0d392c..4904256fac 100644 --- a/crates/services/integration/src/audiobookshelf.rs +++ b/crates/services/integration/src/audiobookshelf.rs @@ -1,36 +1,35 @@ use anyhow::anyhow; -use media_models::{IntegrationMediaSeen, IntegrationMediaCollection}; -use providers::google_books::GoogleBooksService; use reqwest::header::{AUTHORIZATION, HeaderValue}; use rust_decimal_macros::dec; + use application_utils::get_base_http_client; use enums::{MediaLot, MediaSource}; -use crate::integration::Integration; +use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; +use providers::google_books::GoogleBooksService; use specific_models::audiobookshelf; +use crate::integration::Integration; + pub struct AudiobookshelfIntegration { base_url: String, access_token: String, - isbn_service: GoogleBooksService + isbn_service: GoogleBooksService, } impl AudiobookshelfIntegration { - pub fn new( - base_url: String, - access_token: String, - isbn_service: GoogleBooksService - ) -> Self - { + pub fn new(base_url: String, access_token: String, isbn_service: GoogleBooksService) -> Self { Self { base_url, access_token, - isbn_service + isbn_service, } } } impl Integration for AudiobookshelfIntegration { - async fn progress(&self) -> anyhow::Result<(Vec, Vec)> { + async fn progress( + &self, + ) -> anyhow::Result<(Vec, Vec)> { let client = get_base_http_client( &format!("{}/api/", self.base_url), Some(vec![( @@ -85,41 +84,41 @@ impl Integration for AudiobookshelfIntegration { ) } // else if let Some(itunes_id) = metadata.itunes_id.clone() { - // match &item.recent_episode { - // Some(pe) => { - // let lot = MediaLot::Podcast; - // let source = MediaSource::Itunes; - // let podcast = (self.commit_metadata)(CommitMediaInput { - // identifier: itunes_id.clone(), - // lot, - // source, - // force_update: None, - // }).await.map_err(|e| anyhow!("Failed to commit metadata: {:?}", e))?; - // match podcast - // .podcast_specifics - // .and_then(|p| p.episode_by_name(&pe.title)) - // { - // Some(episode) => ( - // format!("{}/{}", item.id, pe.id), - // itunes_id, - // lot, - // source, - // Some(episode), - // ), - // _ => { - // tracing::debug!( - // "No podcast found for iTunes ID {:#?}", - // itunes_id - // ); - // continue; - // } - // } - // } - // _ => { - // tracing::debug!("No recent episode found for item {:#?}", item); - // continue; - // } - // } + // match &item.recent_episode { + // Some(pe) => { + // let lot = MediaLot::Podcast; + // let source = MediaSource::Itunes; + // let podcast = (self.commit_metadata)(CommitMediaInput { + // identifier: itunes_id.clone(), + // lot, + // source, + // force_update: None, + // }).await.map_err(|e| anyhow!("Failed to commit metadata: {:?}", e))?; + // match podcast + // .podcast_specifics + // .and_then(|p| p.episode_by_name(&pe.title)) + // { + // Some(episode) => ( + // format!("{}/{}", item.id, pe.id), + // itunes_id, + // lot, + // source, + // Some(episode), + // ), + // _ => { + // tracing::debug!( + // "No podcast found for iTunes ID {:#?}", + // itunes_id + // ); + // continue; + // } + // } + // } + // _ => { + // tracing::debug!("No recent episode found for item {:#?}", item); + // continue; + // } + // } // } else { tracing::debug!("No ASIN, ISBN or iTunes ID found for item {:#?}", item); diff --git a/crates/services/integration/src/emby.rs b/crates/services/integration/src/emby.rs index 4d1d649e24..35ec05a46b 100644 --- a/crates/services/integration/src/emby.rs +++ b/crates/services/integration/src/emby.rs @@ -1,8 +1,10 @@ use rust_decimal_macros::dec; use sea_orm::DatabaseConnection; use sea_orm::prelude::async_trait::async_trait; + use enums::{MediaLot, MediaSource}; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; + use crate::integration::Integration; use crate::show_identifier::ShowIdentifier; @@ -45,57 +47,66 @@ mod models { } } -pub struct EmbyIntegration -{ +pub struct EmbyIntegration { payload: String, - db: DatabaseConnection + db: DatabaseConnection, } impl EmbyIntegration { pub const fn new(payload: String, db: DatabaseConnection) -> Self { - Self { - payload, - db - } + Self { payload, db } } async fn emby_progress( - &self + &self, ) -> anyhow::Result<(Vec, Vec)> { let payload: models::EmbyWebhookPayload = serde_json::from_str(&self.payload)?; - let runtime = payload.item.run_time_ticks.ok_or_else(|| anyhow::anyhow!("No run time associated with this media"))?; - let position = payload.playback_info.position_ticks.ok_or_else(|| anyhow::anyhow!("No position associated with this media"))?; + let runtime = payload + .item + .run_time_ticks + .ok_or_else(|| anyhow::anyhow!("No run time associated with this media"))?; + let position = payload + .playback_info + .position_ticks + .ok_or_else(|| anyhow::anyhow!("No position associated with this media"))?; - let (identifier, lot) = match payload.item.item_type.as_str() { - "Movie" => { - let id = payload.item.provider_ids.tmdb - .as_ref() - .ok_or_else(|| anyhow::anyhow!("No TMDb ID associated with this media"))?; - (id.clone(), MediaLot::Movie) - }, - "Episode" => { - let series_name = payload.item.series_name - .as_ref() - .ok_or_else(|| anyhow::anyhow!("No series name associated with this media"))?; - let episode_name = payload.item.episode_name - .as_ref() - .ok_or_else(|| anyhow::anyhow!("No episode name associated with this media"))?; - let db_show = self.get_show_by_episode_identifier(series_name, episode_name).await?; - (db_show.identifier, MediaLot::Show) - }, - _ => anyhow::bail!("Only movies and shows supported"), - }; + let (identifier, lot) = + match payload.item.item_type.as_str() { + "Movie" => { + let id = + payload.item.provider_ids.tmdb.as_ref().ok_or_else(|| { + anyhow::anyhow!("No TMDb ID associated with this media") + })?; + (id.clone(), MediaLot::Movie) + } + "Episode" => { + let series_name = payload.item.series_name.as_ref().ok_or_else(|| { + anyhow::anyhow!("No series name associated with this media") + })?; + let episode_name = payload.item.episode_name.as_ref().ok_or_else(|| { + anyhow::anyhow!("No episode name associated with this media") + })?; + let db_show = self + .get_show_by_episode_identifier(series_name, episode_name) + .await?; + (db_show.identifier, MediaLot::Show) + } + _ => anyhow::bail!("Only movies and shows supported"), + }; - Ok((vec![IntegrationMediaSeen { - identifier, - lot, - source: MediaSource::Tmdb, - progress: position / runtime * dec!(100), - show_season_number: payload.item.season_number, - show_episode_number: payload.item.episode_number, - provider_watched_on: Some("Emby".to_string()), - ..Default::default() - }], vec![])) + Ok(( + vec![IntegrationMediaSeen { + identifier, + lot, + source: MediaSource::Tmdb, + progress: position / runtime * dec!(100), + show_season_number: payload.item.season_number, + show_episode_number: payload.item.episode_number, + provider_watched_on: Some("Emby".to_string()), + ..Default::default() + }], + vec![], + )) } } @@ -107,7 +118,9 @@ impl ShowIdentifier for EmbyIntegration { } impl Integration for EmbyIntegration { - async fn progress(&self) -> anyhow::Result<(Vec, Vec)> { + async fn progress( + &self, + ) -> anyhow::Result<(Vec, Vec)> { self.emby_progress().await } } diff --git a/crates/services/integration/src/integration.rs b/crates/services/integration/src/integration.rs index c4b4208c3f..d3dfaa3346 100644 --- a/crates/services/integration/src/integration.rs +++ b/crates/services/integration/src/integration.rs @@ -4,7 +4,9 @@ use crate::IntegrationMediaCollection; use crate::IntegrationMediaSeen; pub trait Integration { - async fn progress(&self) -> Result<(Vec, Vec)>; + async fn progress( + &self, + ) -> Result<(Vec, Vec)>; } pub trait PushIntegration { diff --git a/crates/services/integration/src/integration_type.rs b/crates/services/integration/src/integration_type.rs index d5135e6f24..dfc1f6495a 100644 --- a/crates/services/integration/src/integration_type.rs +++ b/crates/services/integration/src/integration_type.rs @@ -9,5 +9,5 @@ pub enum IntegrationType { Audiobookshelf(String, String, GoogleBooksService), Kodi(String), Sonarr(String, String, i32, String, String), - Radarr(String, String, i32, String, String) + Radarr(String, String, i32, String, String), } diff --git a/crates/services/integration/src/jellyfin.rs b/crates/services/integration/src/jellyfin.rs index b15eb05e64..4df94c4deb 100644 --- a/crates/services/integration/src/jellyfin.rs +++ b/crates/services/integration/src/jellyfin.rs @@ -48,31 +48,42 @@ mod models { } } -pub struct JellyfinIntegration -{ +pub struct JellyfinIntegration { payload: String, } impl JellyfinIntegration { pub const fn new(payload: String) -> Self { - Self { - payload - } + Self { payload } } async fn jellyfin_progress( - &self + &self, ) -> Result<(Vec, Vec)> { let payload = serde_json::from_str::(&self.payload)?; - let identifier = payload.item.provider_ids.tmdb.as_ref() - .or_else(|| payload.series.as_ref().and_then(|s| s.provider_ids.tmdb.as_ref())) + let identifier = payload + .item + .provider_ids + .tmdb + .as_ref() + .or_else(|| { + payload + .series + .as_ref() + .and_then(|s| s.provider_ids.tmdb.as_ref()) + }) .ok_or_else(|| anyhow::anyhow!("No TMDb ID associated with this media"))? .clone(); - let runtime = payload.item.run_time_ticks + let runtime = payload + .item + .run_time_ticks .ok_or_else(|| anyhow::anyhow!("No run time associated with this media"))?; - let position = payload.session.play_state.position_ticks + let position = payload + .session + .play_state + .position_ticks .ok_or_else(|| anyhow::anyhow!("No position associated with this media"))?; let lot = match payload.item.item_type.as_str() { @@ -81,21 +92,26 @@ impl JellyfinIntegration { _ => bail!("Only movies and shows supported"), }; - Ok((vec![IntegrationMediaSeen { - identifier, - lot, - source: MediaSource::Tmdb, - progress: position / runtime * dec!(100), - show_season_number: payload.item.season_number, - show_episode_number: payload.item.episode_number, - provider_watched_on: Some("Jellyfin".to_string()), - ..Default::default() - }], vec![])) + Ok(( + vec![IntegrationMediaSeen { + identifier, + lot, + source: MediaSource::Tmdb, + progress: position / runtime * dec!(100), + show_season_number: payload.item.season_number, + show_episode_number: payload.item.episode_number, + provider_watched_on: Some("Jellyfin".to_string()), + ..Default::default() + }], + vec![], + )) } } impl Integration for JellyfinIntegration { - async fn progress(&self) -> Result<(Vec, Vec)> { + async fn progress( + &self, + ) -> Result<(Vec, Vec)> { self.jellyfin_progress().await } } diff --git a/crates/services/integration/src/kodi.rs b/crates/services/integration/src/kodi.rs index 8077a11912..fe55abe57c 100644 --- a/crates/services/integration/src/kodi.rs +++ b/crates/services/integration/src/kodi.rs @@ -1,20 +1,20 @@ use anyhow::bail; + use enums::MediaSource; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; + use crate::integration::Integration; pub struct KodiIntegration { - payload: String + payload: String, } impl KodiIntegration { pub const fn new(payload: String) -> Self { - Self { - payload - } + Self { payload } } async fn kodi_progress( - &self + &self, ) -> anyhow::Result<(Vec, Vec)> { let mut payload = match serde_json::from_str::(&self.payload) { Ok(val) => val, @@ -27,7 +27,9 @@ impl KodiIntegration { } impl Integration for KodiIntegration { - async fn progress(&self) -> anyhow::Result<(Vec, Vec)> { + async fn progress( + &self, + ) -> anyhow::Result<(Vec, Vec)> { self.kodi_progress().await } } diff --git a/crates/services/integration/src/komga.rs b/crates/services/integration/src/komga.rs index e93cac39f7..26bd93c22d 100644 --- a/crates/services/integration/src/komga.rs +++ b/crates/services/integration/src/komga.rs @@ -193,7 +193,13 @@ pub struct KomgaIntegration { } impl KomgaIntegration { - pub fn new(base_url: String, username: String, password: String, provider: MediaSource, db: DatabaseConnection) -> Self { + pub fn new( + base_url: String, + username: String, + password: String, + provider: MediaSource, + db: DatabaseConnection, + ) -> Self { Self { base_url, username, @@ -337,30 +343,16 @@ impl KomgaIntegration { Decimal::from_f64(percentage).unwrap_or(Decimal::zero()) } - async fn process_events( - &self, - data: komga_events::Data, - ) -> Option { + async fn process_events(&self, data: komga_events::Data) -> Option { let client = get_base_http_client(&format!("{}/api/v1/", self.base_url), None); - let book: komga_book::Item = self.fetch_api( - &client, - "books", - &data.book_id, - ) - .await - .ok()?; - let series: komga_series::Item = self.fetch_api( - &client, - "series", - &book.series_id, - ) + let book: komga_book::Item = self.fetch_api(&client, "books", &data.book_id).await.ok()?; + let series: komga_series::Item = self + .fetch_api(&client, "series", &book.series_id) .await .ok()?; - let (source, id) = self.find_provider_and_id(&series) - .await - .ok()?; + let (source, id) = self.find_provider_and_id(&series).await.ok()?; let Some(id) = id else { tracing::debug!( @@ -382,7 +374,7 @@ impl KomgaIntegration { } async fn komga_progress( - &self + &self, ) -> Result<(Vec, Vec)> { // DEV: This object needs global lifetime so we can continue to use the receiver If // we ever create more SSE Objects we may want to implement a higher level @@ -408,13 +400,8 @@ impl KomgaIntegration { mutex_task.get_or_init(|| { tokio::spawn(async move { - if let Err(e) = Self::sse_listener( - tx, - base_url, - komga_username, - komga_password, - ) - .await + if let Err(e) = + Self::sse_listener(tx, base_url, komga_username, komga_password).await { tracing::error!("SSE listener error: {}", e); } @@ -432,9 +419,8 @@ impl KomgaIntegration { tracing::debug!("Received event {:?}", event); match unique_media_items.entry(event.book_id.clone()) { Entry::Vacant(entry) => { - if let Some(processed_event) = self - .process_events(event.clone()) - .await + if let Some(processed_event) = + self.process_events(event.clone()).await { entry.insert(processed_event); } else { @@ -465,7 +451,9 @@ impl KomgaIntegration { } impl Integration for KomgaIntegration { - async fn progress(&self) -> Result<(Vec, Vec)> { + async fn progress( + &self, + ) -> Result<(Vec, Vec)> { self.komga_progress().await } } diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index f0a209b781..d6533f3a81 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -1,35 +1,28 @@ use anyhow::Result; +use sea_orm::DatabaseConnection; + use database_models::metadata; use enums::{MediaLot, MediaSource}; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; -use sea_orm::DatabaseConnection; use crate::{ - integration::Integration, - integration_type::IntegrationType, - komga::KomgaIntegration + audiobookshelf::AudiobookshelfIntegration, emby::EmbyIntegration, integration::Integration, + integration::PushIntegration, integration_type::IntegrationType, jellyfin::JellyfinIntegration, + kodi::KodiIntegration, komga::KomgaIntegration, plex::PlexIntegration, + radarr::RadarrIntegration, sonarr::SonarrIntegration, }; -use crate::audiobookshelf::AudiobookshelfIntegration; -use crate::emby::EmbyIntegration; -use crate::integration::PushIntegration; -use crate::jellyfin::JellyfinIntegration; -use crate::kodi::KodiIntegration; -use crate::plex::PlexIntegration; -use crate::radarr::RadarrIntegration; -use crate::sonarr::SonarrIntegration; -pub mod integration_type; +mod audiobookshelf; +mod emby; mod integration; - -mod komga; +pub mod integration_type; mod jellyfin; -mod emby; -mod show_identifier; -mod plex; -mod audiobookshelf; mod kodi; -mod sonarr; +mod komga; +mod plex; mod radarr; +mod show_identifier; +mod sonarr; #[derive(Debug)] pub struct IntegrationService { @@ -47,13 +40,14 @@ impl IntegrationService { sonarr_api_key, sonarr_profile_id, sonarr_root_folder_path, - tvdb_id) => { + tvdb_id, + ) => { let sonarr = SonarrIntegration::new( sonarr_base_url, sonarr_api_key, sonarr_profile_id, sonarr_root_folder_path, - tvdb_id + tvdb_id, ); sonarr.push().await } @@ -62,25 +56,29 @@ impl IntegrationService { radarr_api_key, radarr_profile_id, radarr_root_folder_path, - tmdb_id) => { + tmdb_id, + ) => { let radarr = RadarrIntegration::new( radarr_base_url, radarr_api_key, radarr_profile_id, radarr_root_folder_path, - tmdb_id + tmdb_id, ); radarr.push().await } - _ => Ok(()) + _ => Ok(()), } } - pub async fn process_progress(&self, integration_type: IntegrationType) - -> Result<(Vec, Vec)> { + pub async fn process_progress( + &self, + integration_type: IntegrationType, + ) -> Result<(Vec, Vec)> { match integration_type { IntegrationType::Komga(base_url, username, password, provider) => { - let komga = KomgaIntegration::new(base_url, username, password, provider, self.db.clone()); + let komga = + KomgaIntegration::new(base_url, username, password, provider, self.db.clone()); komga.progress().await } IntegrationType::Jellyfin(payload) => { @@ -96,14 +94,15 @@ impl IntegrationService { plex.progress().await } IntegrationType::Audiobookshelf(base_url, access_token, isbn_service) => { - let audiobookshelf = AudiobookshelfIntegration::new(base_url, access_token, isbn_service); + let audiobookshelf = + AudiobookshelfIntegration::new(base_url, access_token, isbn_service); audiobookshelf.progress().await } IntegrationType::Kodi(payload) => { let kodi = KodiIntegration::new(payload); kodi.progress().await } - _ => Ok((vec![],vec![])) + _ => Ok((vec![], vec![])), } } } diff --git a/crates/services/integration/src/plex.rs b/crates/services/integration/src/plex.rs index daa1485411..e82a8af165 100644 --- a/crates/services/integration/src/plex.rs +++ b/crates/services/integration/src/plex.rs @@ -4,10 +4,11 @@ use regex::Regex; use rust_decimal::Decimal; use rust_decimal_macros::dec; use sea_orm::DatabaseConnection; + use enums::{MediaLot, MediaSource}; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; -use crate::integration::Integration; -use crate::show_identifier::ShowIdentifier; + +use crate::{integration::Integration, show_identifier::ShowIdentifier}; mod models { use rust_decimal::Decimal; @@ -62,7 +63,7 @@ impl PlexIntegration { Self { payload, plex_user, - db + db, } } @@ -92,7 +93,9 @@ impl PlexIntegration { "movie" => Ok((identifier.to_owned(), MediaLot::Movie)), "episode" => { let series_name = metadata.show_name.as_ref().context("Show name missing")?; - let db_show = self.get_show_by_episode_identifier(series_name, identifier).await?; + let db_show = self + .get_show_by_episode_identifier(series_name, identifier) + .await?; Ok((db_show.identifier, MediaLot::Show)) } _ => bail!("Only movies and shows supported"), @@ -108,7 +111,7 @@ impl PlexIntegration { } async fn plex_progress( - &self + &self, ) -> anyhow::Result<(Vec, Vec)> { tracing::debug!("Processing Plex payload {:#?}", self.payload); @@ -132,16 +135,19 @@ impl PlexIntegration { let (identifier, lot) = self.get_media_info(&payload.metadata, identifier).await?; let progress = Self::calculate_progress(&payload)?; - Ok((vec![IntegrationMediaSeen { - identifier, - lot, - source: MediaSource::Tmdb, - progress, - provider_watched_on: Some("Plex".to_string()), - show_season_number: payload.metadata.season_number, - show_episode_number: payload.metadata.episode_number, - ..Default::default() - }], vec![])) + Ok(( + vec![IntegrationMediaSeen { + identifier, + lot, + source: MediaSource::Tmdb, + progress, + provider_watched_on: Some("Plex".to_string()), + show_season_number: payload.metadata.season_number, + show_episode_number: payload.metadata.episode_number, + ..Default::default() + }], + vec![], + )) } } @@ -153,7 +159,9 @@ impl ShowIdentifier for PlexIntegration { } impl Integration for PlexIntegration { - async fn progress(&self) -> anyhow::Result<(Vec, Vec)> { + async fn progress( + &self, + ) -> anyhow::Result<(Vec, Vec)> { self.plex_progress().await } } diff --git a/crates/services/integration/src/radarr.rs b/crates/services/integration/src/radarr.rs index d450555003..1e22628b44 100644 --- a/crates/services/integration/src/radarr.rs +++ b/crates/services/integration/src/radarr.rs @@ -1,4 +1,3 @@ -use crate::integration::PushIntegration; use radarr_api_rs::{ apis::{ configuration::{ApiKey as RadarrApiKey, Configuration as RadarrConfiguration}, @@ -6,8 +5,11 @@ use radarr_api_rs::{ }, models::{AddMovieOptions as RadarrAddMovieOptions, MovieResource as RadarrMovieResource}, }; + use traits::TraceOk; +use crate::integration::PushIntegration; + pub struct RadarrIntegration { radarr_base_url: String, radarr_api_key: String, @@ -29,13 +31,11 @@ impl RadarrIntegration { radarr_api_key, radarr_profile_id, radarr_root_folder_path, - tmdb_id + tmdb_id, } } - pub async fn radarr_push( - &self - ) -> anyhow::Result<()> { + pub async fn radarr_push(&self) -> anyhow::Result<()> { let mut configuration = RadarrConfiguration::new(); configuration.base_path = self.radarr_base_url.clone(); configuration.api_key = Some(RadarrApiKey { diff --git a/crates/services/integration/src/show_identifier.rs b/crates/services/integration/src/show_identifier.rs index 54ad35135f..d04fb8e89e 100644 --- a/crates/services/integration/src/show_identifier.rs +++ b/crates/services/integration/src/show_identifier.rs @@ -1,10 +1,13 @@ -use sea_orm::{DatabaseConnection, EntityTrait, QueryFilter, ColumnTrait, Condition}; -use sea_query::{Expr, Func, Alias}; -use anyhow::{Result, bail}; -use sea_orm::prelude::async_trait::async_trait; -use sea_query::extension::postgres::PgExpr; +use anyhow::{bail, Result}; +use sea_orm::{ + ColumnTrait, Condition, DatabaseConnection, EntityTrait, prelude::async_trait::async_trait, + QueryFilter, +}; +use sea_query::{Alias, Expr, extension::postgres::PgExpr, Func}; + use database_utils::ilike_sql; -use crate::{metadata, MediaLot, MediaSource}; + +use crate::{MediaLot, MediaSource, metadata}; #[async_trait] pub trait ShowIdentifier { @@ -25,7 +28,7 @@ pub trait ShowIdentifier { Expr::col(metadata::Column::ShowSpecifics), Alias::new("text"), )) - .ilike(ilike_sql(episode)), + .ilike(ilike_sql(episode)), ) .add(Expr::col(metadata::Column::Title).ilike(ilike_sql(series))), ) diff --git a/crates/services/integration/src/sonarr.rs b/crates/services/integration/src/sonarr.rs index 90c8819ad0..ef55bee125 100644 --- a/crates/services/integration/src/sonarr.rs +++ b/crates/services/integration/src/sonarr.rs @@ -1,4 +1,3 @@ -use crate::integration::PushIntegration; use sonarr_api_rs::{ apis::{ configuration::{ApiKey as SonarrApiKey, Configuration as SonarrConfiguration}, @@ -6,8 +5,11 @@ use sonarr_api_rs::{ }, models::{AddSeriesOptions as SonarrAddSeriesOptions, SeriesResource as SonarrSeriesResource}, }; + use traits::TraceOk; +use crate::integration::PushIntegration; + pub struct SonarrIntegration { sonarr_base_url: String, sonarr_api_key: String, @@ -29,13 +31,11 @@ impl SonarrIntegration { sonarr_api_key, sonarr_profile_id, sonarr_root_folder_path, - tvdb_id + tvdb_id, } } - async fn sonarr_push( - &self - ) -> anyhow::Result<()> { + async fn sonarr_push(&self) -> anyhow::Result<()> { let mut configuration = SonarrConfiguration::new(); configuration.base_path = self.sonarr_base_url.clone(); configuration.api_key = Some(SonarrApiKey { diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index e8e7f37911..a8d7536fef 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -8,19 +8,44 @@ use std::{ }; use apalis::prelude::{MemoryStorage, MessageQueue}; -use application_utils::{get_current_date, user_id_from_token}; use argon2::{Argon2, PasswordHash, PasswordVerifier}; use async_graphql::{Enum, Error, Result}; -use background::{ApplicationJob, CoreApplicationJob}; use cached::{DiskCache, IOCached}; use chrono::{Days, Duration as ChronoDuration, NaiveDate, Utc}; +use enum_meta::Meta; +use futures::TryStreamExt; +use itertools::Itertools; +use markdown::{CompileOptions, Options, to_html_with_options as markdown_to_html_opts}; +use nanoid::nanoid; +use openidconnect::{ + AuthenticationFlow, + AuthorizationCode, + core::{CoreClient, CoreResponseType}, CsrfToken, Nonce, reqwest::async_http_client, Scope, TokenResponse, +}; +use rust_decimal::Decimal; +use rust_decimal_macros::dec; +use sea_orm::{ + ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, DatabaseBackend, + DatabaseConnection, DbBackend, EntityTrait, FromQueryResult, ItemsAndPagesNumber, Iterable, + JoinType, ModelTrait, Order, PaginatorTrait, prelude::DateTimeUtc, QueryFilter, QueryOrder, + QuerySelect, QueryTrait, RelationTrait, sea_query::NullOrdering, Statement, TransactionTrait, +}; +use sea_query::{ + Alias, Asterisk, Cond, Condition, Expr, extension::postgres::PgExpr, Func, Iden, OnConflict, + PgFunc, PostgresQueryBuilder, Query, SelectStatement, SimpleExpr, Write, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use application_utils::{get_current_date, user_id_from_token}; +use background::{ApplicationJob, CoreApplicationJob}; use common_models::{ BackendError, BackgroundJob, ChangeCollectionToEntityInput, DefaultCollection, IdAndNamedObject, MediaStateChanged, SearchDetails, SearchInput, StoredUrl, StringIdObject, UserSummaryData, }; use common_utils::{ - get_first_and_last_day_of_month, IsFeatureEnabled, AUTHOR, SHOW_SPECIAL_SEASON_NAMES, TEMP_DIR, + AUTHOR, get_first_and_last_day_of_month, IsFeatureEnabled, SHOW_SPECIAL_SEASON_NAMES, TEMP_DIR, VERSION, }; use database_models::{ @@ -47,7 +72,6 @@ use dependent_models::{ PersonDetails, SearchResults, UserDetailsResult, UserMetadataDetails, UserMetadataGroupDetails, UserPersonDetails, }; -use enum_meta::Meta; use enums::{ EntityLot, IntegrationLot, IntegrationProvider, MediaLot, MediaSource, MetadataToMetadataRelation, NotificationPlatformLot, SeenState, UserLot, UserToMediaReason, @@ -55,21 +79,18 @@ use enums::{ }; use file_storage_service::FileStorageService; use fitness_models::UserUnitSystem; -use futures::TryStreamExt; -use integration_service::{IntegrationService, integration_type::IntegrationType}; -use itertools::Itertools; +use integration_service::{integration_type::IntegrationType, IntegrationService}; use jwt_service::sign; -use markdown::{to_html_with_options as markdown_to_html_opts, CompileOptions, Options}; use media_models::{ - first_metadata_image_as_url, metadata_images_as_urls, AuthUserInput, CollectionContentsInput, - CollectionContentsSortBy, CollectionItem, CommitMediaInput, CommitPersonInput, - CreateCustomMetadataInput, CreateOrUpdateCollectionInput, CreateReviewCommentInput, - CreateUserIntegrationInput, CreateUserNotificationPlatformInput, EntityWithLot, - GenreDetailsInput, GenreListItem, GraphqlCalendarEvent, GraphqlMediaAssets, - GraphqlMetadataDetails, GraphqlMetadataGroup, GraphqlVideoAsset, GroupedCalendarEvent, - ImportOrExportItemReviewComment, IntegrationMediaSeen, LoginError, LoginErrorVariant, - LoginResponse, LoginResult, MediaAssociatedPersonStateChanges, MediaDetails, - MediaGeneralFilter, MediaSortBy, MetadataCreator, MetadataCreatorGroupedByRole, + AuthUserInput, CollectionContentsInput, CollectionContentsSortBy, CollectionItem, + CommitMediaInput, CommitPersonInput, CreateCustomMetadataInput, CreateOrUpdateCollectionInput, + CreateReviewCommentInput, CreateUserIntegrationInput, CreateUserNotificationPlatformInput, + EntityWithLot, first_metadata_image_as_url, GenreDetailsInput, + GenreListItem, GraphqlCalendarEvent, GraphqlMediaAssets, GraphqlMetadataDetails, + GraphqlMetadataGroup, GraphqlVideoAsset, GroupedCalendarEvent, ImportOrExportItemReviewComment, + IntegrationMediaSeen, LoginError, LoginErrorVariant, LoginResponse, + LoginResult, MediaAssociatedPersonStateChanges, MediaDetails, MediaGeneralFilter, + MediaSortBy, metadata_images_as_urls, MetadataCreator, MetadataCreatorGroupedByRole, MetadataFreeCreator, MetadataGroupSearchInput, MetadataGroupSearchItem, MetadataGroupsListInput, MetadataImage, MetadataImageForMediaDetails, MetadataListInput, MetadataPartialDetails, MetadataSearchInput, MetadataSearchItemResponse, MetadataVideo, @@ -91,13 +112,7 @@ use migrations::{ AliasedMetadataGroup, AliasedMetadataToGenre, AliasedPerson, AliasedSeen, AliasedUser, AliasedUserToCollection, AliasedUserToEntity, }; -use nanoid::nanoid; use notification_service::send_notification; -use openidconnect::{ - core::{CoreClient, CoreResponseType}, - reqwest::async_http_client, - AuthenticationFlow, AuthorizationCode, CsrfToken, Nonce, Scope, TokenResponse, -}; use providers::{ anilist::{AnilistAnimeService, AnilistMangaService, AnilistService, NonMediaAnilistService}, audible::AudibleService, @@ -111,25 +126,11 @@ use providers::{ tmdb::{NonMediaTmdbService, TmdbMovieService, TmdbService, TmdbShowService}, vndb::VndbService, }; -use rust_decimal::Decimal; -use rust_decimal_macros::dec; -use sea_orm::{ - prelude::DateTimeUtc, sea_query::NullOrdering, ActiveModelTrait, ActiveValue, ColumnTrait, - ConnectionTrait, DatabaseBackend, DatabaseConnection, DbBackend, EntityTrait, FromQueryResult, - ItemsAndPagesNumber, Iterable, JoinType, ModelTrait, Order, PaginatorTrait, QueryFilter, - QueryOrder, QuerySelect, QueryTrait, RelationTrait, Statement, TransactionTrait, -}; -use sea_query::{ - extension::postgres::PgExpr, Alias, Asterisk, Cond, Condition, Expr, Func, Iden, OnConflict, - PgFunc, PostgresQueryBuilder, Query, SelectStatement, SimpleExpr, Write, -}; -use serde::{Deserialize, Serialize}; use traits::{MediaProvider, MediaProviderLanguages, TraceOk}; use user_models::{ NotificationPlatformSpecifics, UserGeneralDashboardElement, UserGeneralPreferences, UserPreferences, UserReviewScale, }; -use uuid::Uuid; type Provider = Box<(dyn MediaProvider + Send + Sync)>; @@ -4433,20 +4434,24 @@ impl MiscellaneousService { let response = match integration.provider { IntegrationProvider::Audiobookshelf => { let specifics = integration.clone().provider_specifics.unwrap(); - integration_service.process_progress(IntegrationType::Audiobookshelf( - specifics.audiobookshelf_base_url.unwrap(), - specifics.audiobookshelf_token.unwrap(), - self.get_isbn_service().await.unwrap(), - )).await + integration_service + .process_progress(IntegrationType::Audiobookshelf( + specifics.audiobookshelf_base_url.unwrap(), + specifics.audiobookshelf_token.unwrap(), + self.get_isbn_service().await.unwrap(), + )) + .await } IntegrationProvider::Komga => { let specifics = integration.clone().provider_specifics.unwrap(); - integration_service.process_progress(IntegrationType::Komga( - specifics.komga_base_url.unwrap(), - specifics.komga_username.unwrap(), - specifics.komga_password.unwrap(), - specifics.komga_provider.unwrap(), - )).await + integration_service + .process_progress(IntegrationType::Komga( + specifics.komga_base_url.unwrap(), + specifics.komga_username.unwrap(), + specifics.komga_password.unwrap(), + specifics.komga_provider.unwrap(), + )) + .await } _ => continue, }; @@ -4560,22 +4565,26 @@ impl MiscellaneousService { if let Some(entity_id) = maybe_entity_id { let _push_result = match integration.provider { IntegrationProvider::Radarr => { - integration_service.push(IntegrationType::Radarr( - specifics.radarr_base_url.unwrap(), - specifics.radarr_api_key.unwrap(), - specifics.radarr_profile_id.unwrap(), - specifics.radarr_root_folder_path.unwrap(), - entity_id, - )).await + integration_service + .push(IntegrationType::Radarr( + specifics.radarr_base_url.unwrap(), + specifics.radarr_api_key.unwrap(), + specifics.radarr_profile_id.unwrap(), + specifics.radarr_root_folder_path.unwrap(), + entity_id, + )) + .await } IntegrationProvider::Sonarr => { - integration_service.push(IntegrationType::Sonarr( - specifics.sonarr_base_url.unwrap(), - specifics.sonarr_api_key.unwrap(), - specifics.sonarr_profile_id.unwrap(), - specifics.sonarr_root_folder_path.unwrap(), - entity_id, - )).await + integration_service + .push(IntegrationType::Sonarr( + specifics.sonarr_base_url.unwrap(), + specifics.sonarr_api_key.unwrap(), + specifics.sonarr_profile_id.unwrap(), + specifics.sonarr_root_folder_path.unwrap(), + entity_id, + )) + .await } _ => unreachable!(), }; @@ -4644,30 +4653,42 @@ impl MiscellaneousService { } let service = self.get_integration_service(); let maybe_progress_update = match integration.provider { - IntegrationProvider::Kodi => service.process_progress(IntegrationType::Kodi( - payload.clone() - )).await, - IntegrationProvider::Emby => service.process_progress(IntegrationType::Emby( - payload.clone() - )).await, - IntegrationProvider::Jellyfin => service.process_progress(IntegrationType::Jellyfin( - payload.clone() - )).await, + IntegrationProvider::Kodi => { + service + .process_progress(IntegrationType::Kodi(payload.clone())) + .await + } + IntegrationProvider::Emby => { + service + .process_progress(IntegrationType::Emby(payload.clone())) + .await + } + IntegrationProvider::Jellyfin => { + service + .process_progress(IntegrationType::Jellyfin(payload.clone())) + .await + } IntegrationProvider::Plex => { let specifics = integration.clone().provider_specifics.unwrap(); - service.process_progress(IntegrationType::Plex( - payload.clone(), - specifics.plex_username - )).await - }, + service + .process_progress(IntegrationType::Plex( + payload.clone(), + specifics.plex_username, + )) + .await + } _ => return Err(Error::new("Unsupported integration source".to_owned())), }; match maybe_progress_update { Ok(pu) => { let media_vec = pu.0; - for media in media_vec - { - self.integration_progress_update(&integration, media.clone(), &integration.user_id).await?; + for media in media_vec { + self.integration_progress_update( + &integration, + media.clone(), + &integration.user_id, + ) + .await?; } let mut to_update: integration::ActiveModel = integration.into(); to_update.last_triggered_on = ActiveValue::Set(Some(Utc::now())); From 8ecc8a6a3da8975b6325d846e4f69e20e7b93801 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sun, 18 Aug 2024 10:33:31 +0530 Subject: [PATCH 57/61] chore: run rust formatter --- .../migrations/src/m20230505_create_review.rs | 2 +- crates/models/media/src/lib.rs | 2 +- .../integration/src/audiobookshelf.rs | 2 +- crates/services/integration/src/emby.rs | 2 +- crates/services/integration/src/komga.rs | 2 +- .../integration/src/show_identifier.rs | 6 +-- crates/services/miscellaneous/src/lib.rs | 38 +++++++++---------- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/crates/migrations/src/m20230505_create_review.rs b/crates/migrations/src/m20230505_create_review.rs index 55d51134f3..893459974a 100644 --- a/crates/migrations/src/m20230505_create_review.rs +++ b/crates/migrations/src/m20230505_create_review.rs @@ -1,5 +1,5 @@ -use sea_orm_migration::prelude::*; use enums::Visibility; +use sea_orm_migration::prelude::*; use super::{ m20230410_create_metadata::Metadata, m20230413_create_person::Person, diff --git a/crates/models/media/src/lib.rs b/crates/models/media/src/lib.rs index 25dea12ee3..f42122d466 100644 --- a/crates/models/media/src/lib.rs +++ b/crates/models/media/src/lib.rs @@ -2,7 +2,7 @@ use std::{collections::HashSet, fmt, sync::Arc}; use async_graphql::{Enum, InputObject, InputType, OneofObject, SimpleObject, Union}; use boilermates::boilermates; -use chrono::{DateTime, NaiveDate,NaiveDateTime}; +use chrono::{DateTime, NaiveDate, NaiveDateTime}; use common_models::{ CollectionExtraInformation, IdAndNamedObject, SearchInput, StoredUrl, StringIdObject, }; diff --git a/crates/services/integration/src/audiobookshelf.rs b/crates/services/integration/src/audiobookshelf.rs index 4904256fac..d6748b1a1d 100644 --- a/crates/services/integration/src/audiobookshelf.rs +++ b/crates/services/integration/src/audiobookshelf.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use reqwest::header::{AUTHORIZATION, HeaderValue}; +use reqwest::header::{HeaderValue, AUTHORIZATION}; use rust_decimal_macros::dec; use application_utils::get_base_http_client; diff --git a/crates/services/integration/src/emby.rs b/crates/services/integration/src/emby.rs index 35ec05a46b..07cee6770a 100644 --- a/crates/services/integration/src/emby.rs +++ b/crates/services/integration/src/emby.rs @@ -1,6 +1,6 @@ use rust_decimal_macros::dec; -use sea_orm::DatabaseConnection; use sea_orm::prelude::async_trait::async_trait; +use sea_orm::DatabaseConnection; use enums::{MediaLot, MediaSource}; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; diff --git a/crates/services/integration/src/komga.rs b/crates/services/integration/src/komga.rs index 26bd93c22d..8078e84e33 100644 --- a/crates/services/integration/src/komga.rs +++ b/crates/services/integration/src/komga.rs @@ -8,8 +8,8 @@ use async_graphql::futures_util::StreamExt; use eventsource_stream::Eventsource; use reqwest::Url; use rust_decimal::{ - Decimal, prelude::{FromPrimitive, Zero}, + Decimal, }; use sea_orm::{ColumnTrait, DatabaseConnection, EntityTrait, QueryFilter}; use sea_query::Expr; diff --git a/crates/services/integration/src/show_identifier.rs b/crates/services/integration/src/show_identifier.rs index d04fb8e89e..7993aa171b 100644 --- a/crates/services/integration/src/show_identifier.rs +++ b/crates/services/integration/src/show_identifier.rs @@ -1,13 +1,13 @@ use anyhow::{bail, Result}; use sea_orm::{ - ColumnTrait, Condition, DatabaseConnection, EntityTrait, prelude::async_trait::async_trait, + prelude::async_trait::async_trait, ColumnTrait, Condition, DatabaseConnection, EntityTrait, QueryFilter, }; -use sea_query::{Alias, Expr, extension::postgres::PgExpr, Func}; +use sea_query::{extension::postgres::PgExpr, Alias, Expr, Func}; use database_utils::ilike_sql; -use crate::{MediaLot, MediaSource, metadata}; +use crate::{metadata, MediaLot, MediaSource}; #[async_trait] pub trait ShowIdentifier { diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index a8d7536fef..748255d856 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -15,23 +15,23 @@ use chrono::{Days, Duration as ChronoDuration, NaiveDate, Utc}; use enum_meta::Meta; use futures::TryStreamExt; use itertools::Itertools; -use markdown::{CompileOptions, Options, to_html_with_options as markdown_to_html_opts}; +use markdown::{to_html_with_options as markdown_to_html_opts, CompileOptions, Options}; use nanoid::nanoid; use openidconnect::{ - AuthenticationFlow, - AuthorizationCode, - core::{CoreClient, CoreResponseType}, CsrfToken, Nonce, reqwest::async_http_client, Scope, TokenResponse, + core::{CoreClient, CoreResponseType}, + reqwest::async_http_client, + AuthenticationFlow, AuthorizationCode, CsrfToken, Nonce, Scope, TokenResponse, }; use rust_decimal::Decimal; use rust_decimal_macros::dec; use sea_orm::{ - ActiveModelTrait, ActiveValue, ColumnTrait, ConnectionTrait, DatabaseBackend, - DatabaseConnection, DbBackend, EntityTrait, FromQueryResult, ItemsAndPagesNumber, Iterable, - JoinType, ModelTrait, Order, PaginatorTrait, prelude::DateTimeUtc, QueryFilter, QueryOrder, - QuerySelect, QueryTrait, RelationTrait, sea_query::NullOrdering, Statement, TransactionTrait, + prelude::DateTimeUtc, sea_query::NullOrdering, ActiveModelTrait, ActiveValue, ColumnTrait, + ConnectionTrait, DatabaseBackend, DatabaseConnection, DbBackend, EntityTrait, FromQueryResult, + ItemsAndPagesNumber, Iterable, JoinType, ModelTrait, Order, PaginatorTrait, QueryFilter, + QueryOrder, QuerySelect, QueryTrait, RelationTrait, Statement, TransactionTrait, }; use sea_query::{ - Alias, Asterisk, Cond, Condition, Expr, extension::postgres::PgExpr, Func, Iden, OnConflict, + extension::postgres::PgExpr, Alias, Asterisk, Cond, Condition, Expr, Func, Iden, OnConflict, PgFunc, PostgresQueryBuilder, Query, SelectStatement, SimpleExpr, Write, }; use serde::{Deserialize, Serialize}; @@ -45,7 +45,7 @@ use common_models::{ UserSummaryData, }; use common_utils::{ - AUTHOR, get_first_and_last_day_of_month, IsFeatureEnabled, SHOW_SPECIAL_SEASON_NAMES, TEMP_DIR, + get_first_and_last_day_of_month, IsFeatureEnabled, AUTHOR, SHOW_SPECIAL_SEASON_NAMES, TEMP_DIR, VERSION, }; use database_models::{ @@ -82,15 +82,15 @@ use fitness_models::UserUnitSystem; use integration_service::{integration_type::IntegrationType, IntegrationService}; use jwt_service::sign; use media_models::{ - AuthUserInput, CollectionContentsInput, CollectionContentsSortBy, CollectionItem, - CommitMediaInput, CommitPersonInput, CreateCustomMetadataInput, CreateOrUpdateCollectionInput, - CreateReviewCommentInput, CreateUserIntegrationInput, CreateUserNotificationPlatformInput, - EntityWithLot, first_metadata_image_as_url, GenreDetailsInput, - GenreListItem, GraphqlCalendarEvent, GraphqlMediaAssets, GraphqlMetadataDetails, - GraphqlMetadataGroup, GraphqlVideoAsset, GroupedCalendarEvent, ImportOrExportItemReviewComment, - IntegrationMediaSeen, LoginError, LoginErrorVariant, LoginResponse, - LoginResult, MediaAssociatedPersonStateChanges, MediaDetails, MediaGeneralFilter, - MediaSortBy, metadata_images_as_urls, MetadataCreator, MetadataCreatorGroupedByRole, + first_metadata_image_as_url, metadata_images_as_urls, AuthUserInput, CollectionContentsInput, + CollectionContentsSortBy, CollectionItem, CommitMediaInput, CommitPersonInput, + CreateCustomMetadataInput, CreateOrUpdateCollectionInput, CreateReviewCommentInput, + CreateUserIntegrationInput, CreateUserNotificationPlatformInput, EntityWithLot, + GenreDetailsInput, GenreListItem, GraphqlCalendarEvent, GraphqlMediaAssets, + GraphqlMetadataDetails, GraphqlMetadataGroup, GraphqlVideoAsset, GroupedCalendarEvent, + ImportOrExportItemReviewComment, IntegrationMediaSeen, LoginError, LoginErrorVariant, + LoginResponse, LoginResult, MediaAssociatedPersonStateChanges, MediaDetails, + MediaGeneralFilter, MediaSortBy, MetadataCreator, MetadataCreatorGroupedByRole, MetadataFreeCreator, MetadataGroupSearchInput, MetadataGroupSearchItem, MetadataGroupsListInput, MetadataImage, MetadataImageForMediaDetails, MetadataListInput, MetadataPartialDetails, MetadataSearchInput, MetadataSearchItemResponse, MetadataVideo, From 79c6c05292af163db50777e088c9fe6ff7976bad Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Sun, 18 Aug 2024 10:41:09 +0530 Subject: [PATCH 58/61] refactor(integrations): some import changes --- .../integration/src/audiobookshelf.rs | 11 ++++---- crates/services/integration/src/emby.rs | 17 ++++++------- .../services/integration/src/integration.rs | 9 +++---- crates/services/integration/src/jellyfin.rs | 17 ++++++------- crates/services/integration/src/kodi.rs | 7 +++--- crates/services/integration/src/komga.rs | 15 +++++------ crates/services/integration/src/lib.rs | 25 ++++++++----------- crates/services/integration/src/plex.rs | 15 ++++++----- crates/services/integration/src/radarr.rs | 5 ++-- .../integration/src/show_identifier.rs | 7 +++--- crates/services/integration/src/sonarr.rs | 5 ++-- 11 files changed, 59 insertions(+), 74 deletions(-) diff --git a/crates/services/integration/src/audiobookshelf.rs b/crates/services/integration/src/audiobookshelf.rs index d6748b1a1d..d359f2a7dc 100644 --- a/crates/services/integration/src/audiobookshelf.rs +++ b/crates/services/integration/src/audiobookshelf.rs @@ -1,14 +1,13 @@ use anyhow::anyhow; -use reqwest::header::{HeaderValue, AUTHORIZATION}; -use rust_decimal_macros::dec; - use application_utils::get_base_http_client; use enums::{MediaLot, MediaSource}; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; use providers::google_books::GoogleBooksService; +use reqwest::header::{HeaderValue, AUTHORIZATION}; +use rust_decimal_macros::dec; use specific_models::audiobookshelf; -use crate::integration::Integration; +use super::integration::YankIntegration; pub struct AudiobookshelfIntegration { base_url: String, @@ -26,8 +25,8 @@ impl AudiobookshelfIntegration { } } -impl Integration for AudiobookshelfIntegration { - async fn progress( +impl YankIntegration for AudiobookshelfIntegration { + async fn yank_progress( &self, ) -> anyhow::Result<(Vec, Vec)> { let client = get_base_http_client( diff --git a/crates/services/integration/src/emby.rs b/crates/services/integration/src/emby.rs index 07cee6770a..63ce592565 100644 --- a/crates/services/integration/src/emby.rs +++ b/crates/services/integration/src/emby.rs @@ -1,16 +1,15 @@ +use enums::{MediaLot, MediaSource}; +use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; +use rust_decimal::Decimal; use rust_decimal_macros::dec; use sea_orm::prelude::async_trait::async_trait; use sea_orm::DatabaseConnection; +use serde::{Deserialize, Serialize}; -use enums::{MediaLot, MediaSource}; -use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; - -use crate::integration::Integration; -use crate::show_identifier::ShowIdentifier; +use super::{integration::YankIntegration, show_identifier::ShowIdentifier}; mod models { - use rust_decimal::Decimal; - use serde::{Deserialize, Serialize}; + use super::*; #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "PascalCase")] @@ -117,8 +116,8 @@ impl ShowIdentifier for EmbyIntegration { } } -impl Integration for EmbyIntegration { - async fn progress( +impl YankIntegration for EmbyIntegration { + async fn yank_progress( &self, ) -> anyhow::Result<(Vec, Vec)> { self.emby_progress().await diff --git a/crates/services/integration/src/integration.rs b/crates/services/integration/src/integration.rs index d3dfaa3346..76a34af1b4 100644 --- a/crates/services/integration/src/integration.rs +++ b/crates/services/integration/src/integration.rs @@ -1,14 +1,13 @@ use anyhow::Result; -use crate::IntegrationMediaCollection; -use crate::IntegrationMediaSeen; +use super::{IntegrationMediaCollection, IntegrationMediaSeen}; -pub trait Integration { - async fn progress( +pub trait YankIntegration { + async fn yank_progress( &self, ) -> Result<(Vec, Vec)>; } pub trait PushIntegration { - async fn push(&self) -> Result<()>; + async fn push_progress(&self) -> Result<()>; } diff --git a/crates/services/integration/src/jellyfin.rs b/crates/services/integration/src/jellyfin.rs index 4df94c4deb..4bc033aa21 100644 --- a/crates/services/integration/src/jellyfin.rs +++ b/crates/services/integration/src/jellyfin.rs @@ -1,15 +1,14 @@ -use anyhow::bail; -use anyhow::Result; -use rust_decimal_macros::dec; - +use anyhow::{bail, Result}; use enums::{MediaLot, MediaSource}; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; +use rust_decimal::Decimal; +use rust_decimal_macros::dec; +use serde::{Deserialize, Serialize}; -use crate::integration::Integration; +use super::integration::YankIntegration; mod models { - use rust_decimal::Decimal; - use serde::{Deserialize, Serialize}; + use super::*; #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "PascalCase")] @@ -108,8 +107,8 @@ impl JellyfinIntegration { } } -impl Integration for JellyfinIntegration { - async fn progress( +impl YankIntegration for JellyfinIntegration { + async fn yank_progress( &self, ) -> Result<(Vec, Vec)> { self.jellyfin_progress().await diff --git a/crates/services/integration/src/kodi.rs b/crates/services/integration/src/kodi.rs index fe55abe57c..8ec718b5ee 100644 --- a/crates/services/integration/src/kodi.rs +++ b/crates/services/integration/src/kodi.rs @@ -1,9 +1,8 @@ use anyhow::bail; - use enums::MediaSource; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; -use crate::integration::Integration; +use super::integration::YankIntegration; pub struct KodiIntegration { payload: String, @@ -26,8 +25,8 @@ impl KodiIntegration { } } -impl Integration for KodiIntegration { - async fn progress( +impl YankIntegration for KodiIntegration { + async fn yank_progress( &self, ) -> anyhow::Result<(Vec, Vec)> { self.kodi_progress().await diff --git a/crates/services/integration/src/komga.rs b/crates/services/integration/src/komga.rs index 8078e84e33..299dc00711 100644 --- a/crates/services/integration/src/komga.rs +++ b/crates/services/integration/src/komga.rs @@ -4,7 +4,10 @@ use std::{ }; use anyhow::{anyhow, Context, Result}; +use application_utils::get_base_http_client; use async_graphql::futures_util::StreamExt; +use database_models::{metadata, prelude::Metadata}; +use enums::{MediaLot, MediaSource}; use eventsource_stream::Eventsource; use reqwest::Url; use rust_decimal::{ @@ -16,13 +19,7 @@ use sea_query::Expr; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tokio::sync::{mpsc, mpsc::error::TryRecvError, mpsc::UnboundedReceiver}; -use application_utils::get_base_http_client; -use database_models::{metadata, prelude::Metadata}; -use enums::{MediaLot, MediaSource}; - -use crate::integration::Integration; - -use super::{IntegrationMediaCollection, IntegrationMediaSeen}; +use super::{integration::YankIntegration, IntegrationMediaCollection, IntegrationMediaSeen}; mod komga_book { use super::*; @@ -450,8 +447,8 @@ impl KomgaIntegration { } } -impl Integration for KomgaIntegration { - async fn progress( +impl YankIntegration for KomgaIntegration { + async fn yank_progress( &self, ) -> Result<(Vec, Vec)> { self.komga_progress().await diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index d6533f3a81..61860a3c26 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -1,13 +1,10 @@ use anyhow::Result; -use sea_orm::DatabaseConnection; - -use database_models::metadata; -use enums::{MediaLot, MediaSource}; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; +use sea_orm::DatabaseConnection; use crate::{ - audiobookshelf::AudiobookshelfIntegration, emby::EmbyIntegration, integration::Integration, - integration::PushIntegration, integration_type::IntegrationType, jellyfin::JellyfinIntegration, + audiobookshelf::AudiobookshelfIntegration, emby::EmbyIntegration, integration::PushIntegration, + integration::YankIntegration, integration_type::IntegrationType, jellyfin::JellyfinIntegration, kodi::KodiIntegration, komga::KomgaIntegration, plex::PlexIntegration, radarr::RadarrIntegration, sonarr::SonarrIntegration, }; @@ -49,7 +46,7 @@ impl IntegrationService { sonarr_root_folder_path, tvdb_id, ); - sonarr.push().await + sonarr.push_progress().await } IntegrationType::Radarr( radarr_base_url, @@ -65,7 +62,7 @@ impl IntegrationService { radarr_root_folder_path, tmdb_id, ); - radarr.push().await + radarr.push_progress().await } _ => Ok(()), } @@ -79,28 +76,28 @@ impl IntegrationService { IntegrationType::Komga(base_url, username, password, provider) => { let komga = KomgaIntegration::new(base_url, username, password, provider, self.db.clone()); - komga.progress().await + komga.yank_progress().await } IntegrationType::Jellyfin(payload) => { let jellyfin = JellyfinIntegration::new(payload); - jellyfin.progress().await + jellyfin.yank_progress().await } IntegrationType::Emby(payload) => { let emby = EmbyIntegration::new(payload, self.db.clone()); - emby.progress().await + emby.yank_progress().await } IntegrationType::Plex(payload, plex_user) => { let plex = PlexIntegration::new(payload, plex_user, self.db.clone()); - plex.progress().await + plex.yank_progress().await } IntegrationType::Audiobookshelf(base_url, access_token, isbn_service) => { let audiobookshelf = AudiobookshelfIntegration::new(base_url, access_token, isbn_service); - audiobookshelf.progress().await + audiobookshelf.yank_progress().await } IntegrationType::Kodi(payload) => { let kodi = KodiIntegration::new(payload); - kodi.progress().await + kodi.yank_progress().await } _ => Ok((vec![], vec![])), } diff --git a/crates/services/integration/src/plex.rs b/crates/services/integration/src/plex.rs index e82a8af165..65de711f13 100644 --- a/crates/services/integration/src/plex.rs +++ b/crates/services/integration/src/plex.rs @@ -1,18 +1,17 @@ use anyhow::{bail, Context}; use async_graphql::async_trait::async_trait; +use enums::{MediaLot, MediaSource}; +use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; use regex::Regex; use rust_decimal::Decimal; use rust_decimal_macros::dec; use sea_orm::DatabaseConnection; +use serde::{Deserialize, Serialize}; -use enums::{MediaLot, MediaSource}; -use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; - -use crate::{integration::Integration, show_identifier::ShowIdentifier}; +use super::{integration::YankIntegration, show_identifier::ShowIdentifier}; mod models { - use rust_decimal::Decimal; - use serde::{Deserialize, Serialize}; + use super::*; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct PlexWebhookMetadataGuid { @@ -158,8 +157,8 @@ impl ShowIdentifier for PlexIntegration { } } -impl Integration for PlexIntegration { - async fn progress( +impl YankIntegration for PlexIntegration { + async fn yank_progress( &self, ) -> anyhow::Result<(Vec, Vec)> { self.plex_progress().await diff --git a/crates/services/integration/src/radarr.rs b/crates/services/integration/src/radarr.rs index 1e22628b44..3dc0aa7873 100644 --- a/crates/services/integration/src/radarr.rs +++ b/crates/services/integration/src/radarr.rs @@ -5,10 +5,9 @@ use radarr_api_rs::{ }, models::{AddMovieOptions as RadarrAddMovieOptions, MovieResource as RadarrMovieResource}, }; - use traits::TraceOk; -use crate::integration::PushIntegration; +use super::integration::PushIntegration; pub struct RadarrIntegration { radarr_base_url: String, @@ -59,7 +58,7 @@ impl RadarrIntegration { } impl PushIntegration for RadarrIntegration { - async fn push(&self) -> anyhow::Result<()> { + async fn push_progress(&self) -> anyhow::Result<()> { self.radarr_push().await } } diff --git a/crates/services/integration/src/show_identifier.rs b/crates/services/integration/src/show_identifier.rs index 7993aa171b..c8e8929150 100644 --- a/crates/services/integration/src/show_identifier.rs +++ b/crates/services/integration/src/show_identifier.rs @@ -1,14 +1,13 @@ use anyhow::{bail, Result}; +use database_models::metadata; +use database_utils::ilike_sql; +use enums::{MediaLot, MediaSource}; use sea_orm::{ prelude::async_trait::async_trait, ColumnTrait, Condition, DatabaseConnection, EntityTrait, QueryFilter, }; use sea_query::{extension::postgres::PgExpr, Alias, Expr, Func}; -use database_utils::ilike_sql; - -use crate::{metadata, MediaLot, MediaSource}; - #[async_trait] pub trait ShowIdentifier { fn get_db(&self) -> &DatabaseConnection; diff --git a/crates/services/integration/src/sonarr.rs b/crates/services/integration/src/sonarr.rs index ef55bee125..3020996984 100644 --- a/crates/services/integration/src/sonarr.rs +++ b/crates/services/integration/src/sonarr.rs @@ -5,10 +5,9 @@ use sonarr_api_rs::{ }, models::{AddSeriesOptions as SonarrAddSeriesOptions, SeriesResource as SonarrSeriesResource}, }; - use traits::TraceOk; -use crate::integration::PushIntegration; +use super::integration::PushIntegration; pub struct SonarrIntegration { sonarr_base_url: String, @@ -61,7 +60,7 @@ impl SonarrIntegration { } impl PushIntegration for SonarrIntegration { - async fn push(&self) -> anyhow::Result<()> { + async fn push_progress(&self) -> anyhow::Result<()> { self.sonarr_push().await } } From 67150167b698e35fbc5e65fdd437ff1651825a72 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 23:06:27 -0700 Subject: [PATCH 59/61] Finished implementing Audiobookshelf Introduced YankIntegrationWithCommit trait to address issue with passing the closure through a staticly allocated enum. Renamed the integration file to integration trait and moved all integrations traits to this file Changed visibility of all integrations to be pub(crate) since they dont need to be accessible anywhere else. Introduced proper error handling for improper integration types --- .../integration/src/audiobookshelf.rs | 96 ++++++++++--------- crates/services/integration/src/emby.rs | 4 +- .../services/integration/src/integration.rs | 13 --- ...how_identifier.rs => integration_trait.rs} | 25 ++++- crates/services/integration/src/jellyfin.rs | 4 +- crates/services/integration/src/kodi.rs | 4 +- crates/services/integration/src/komga.rs | 4 +- crates/services/integration/src/lib.rs | 43 ++++++--- crates/services/integration/src/plex.rs | 4 +- crates/services/integration/src/radarr.rs | 6 +- crates/services/integration/src/sonarr.rs | 4 +- crates/services/miscellaneous/src/lib.rs | 6 +- 12 files changed, 125 insertions(+), 88 deletions(-) delete mode 100644 crates/services/integration/src/integration.rs rename crates/services/integration/src/{show_identifier.rs => integration_trait.rs} (65%) diff --git a/crates/services/integration/src/audiobookshelf.rs b/crates/services/integration/src/audiobookshelf.rs index d359f2a7dc..963dba6daa 100644 --- a/crates/services/integration/src/audiobookshelf.rs +++ b/crates/services/integration/src/audiobookshelf.rs @@ -1,15 +1,19 @@ +use std::future::Future; + use anyhow::anyhow; use application_utils::get_base_http_client; +use async_graphql::Result as GqlResult; +use database_models::metadata; use enums::{MediaLot, MediaSource}; -use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; +use media_models::{CommitMediaInput, IntegrationMediaCollection, IntegrationMediaSeen}; use providers::google_books::GoogleBooksService; use reqwest::header::{HeaderValue, AUTHORIZATION}; use rust_decimal_macros::dec; use specific_models::audiobookshelf; -use super::integration::YankIntegration; +use super::integration_trait::YankIntegrationWithCommit; -pub struct AudiobookshelfIntegration { +pub(crate) struct AudiobookshelfIntegration { base_url: String, access_token: String, isbn_service: GoogleBooksService, @@ -25,10 +29,14 @@ impl AudiobookshelfIntegration { } } -impl YankIntegration for AudiobookshelfIntegration { - async fn yank_progress( +impl YankIntegrationWithCommit for AudiobookshelfIntegration { + async fn yank_progress( &self, - ) -> anyhow::Result<(Vec, Vec)> { + commit_metadata: impl Fn(CommitMediaInput) -> F, + ) -> anyhow::Result<(Vec, Vec)> + where + F: Future> + { let client = get_base_http_client( &format!("{}/api/", self.base_url), Some(vec![( @@ -82,43 +90,45 @@ impl YankIntegration for AudiobookshelfIntegration { None, ) } - // else if let Some(itunes_id) = metadata.itunes_id.clone() { - // match &item.recent_episode { - // Some(pe) => { - // let lot = MediaLot::Podcast; - // let source = MediaSource::Itunes; - // let podcast = (self.commit_metadata)(CommitMediaInput { - // identifier: itunes_id.clone(), - // lot, - // source, - // force_update: None, - // }).await.map_err(|e| anyhow!("Failed to commit metadata: {:?}", e))?; - // match podcast - // .podcast_specifics - // .and_then(|p| p.episode_by_name(&pe.title)) - // { - // Some(episode) => ( - // format!("{}/{}", item.id, pe.id), - // itunes_id, - // lot, - // source, - // Some(episode), - // ), - // _ => { - // tracing::debug!( - // "No podcast found for iTunes ID {:#?}", - // itunes_id - // ); - // continue; - // } - // } - // } - // _ => { - // tracing::debug!("No recent episode found for item {:#?}", item); - // continue; - // } - // } - // } + else if let Some(itunes_id) = metadata.itunes_id.clone() { + match &item.recent_episode { + Some(pe) => { + let lot = MediaLot::Podcast; + let source = MediaSource::Itunes; + let podcast = commit_metadata(CommitMediaInput { + identifier: itunes_id.clone(), + lot, + source, + ..Default::default() + }) + .await + .unwrap(); + match podcast + .podcast_specifics + .and_then(|p| p.episode_by_name(&pe.title)) + { + Some(episode) => ( + format!("{}/{}", item.id, pe.id), + itunes_id, + lot, + source, + Some(episode), + ), + _ => { + tracing::debug!( + "No podcast found for iTunes ID {:#?}", + itunes_id + ); + continue; + } + } + } + _ => { + tracing::debug!("No recent episode found for item {:#?}", item); + continue; + } + } + } else { tracing::debug!("No ASIN, ISBN or iTunes ID found for item {:#?}", item); continue; diff --git a/crates/services/integration/src/emby.rs b/crates/services/integration/src/emby.rs index 63ce592565..368c0600c5 100644 --- a/crates/services/integration/src/emby.rs +++ b/crates/services/integration/src/emby.rs @@ -6,7 +6,7 @@ use sea_orm::prelude::async_trait::async_trait; use sea_orm::DatabaseConnection; use serde::{Deserialize, Serialize}; -use super::{integration::YankIntegration, show_identifier::ShowIdentifier}; +use super::{integration_trait::ShowIdentifier, integration_trait::YankIntegration}; mod models { use super::*; @@ -46,7 +46,7 @@ mod models { } } -pub struct EmbyIntegration { +pub(crate) struct EmbyIntegration { payload: String, db: DatabaseConnection, } diff --git a/crates/services/integration/src/integration.rs b/crates/services/integration/src/integration.rs deleted file mode 100644 index 76a34af1b4..0000000000 --- a/crates/services/integration/src/integration.rs +++ /dev/null @@ -1,13 +0,0 @@ -use anyhow::Result; - -use super::{IntegrationMediaCollection, IntegrationMediaSeen}; - -pub trait YankIntegration { - async fn yank_progress( - &self, - ) -> Result<(Vec, Vec)>; -} - -pub trait PushIntegration { - async fn push_progress(&self) -> Result<()>; -} diff --git a/crates/services/integration/src/show_identifier.rs b/crates/services/integration/src/integration_trait.rs similarity index 65% rename from crates/services/integration/src/show_identifier.rs rename to crates/services/integration/src/integration_trait.rs index c8e8929150..a2b17df7a9 100644 --- a/crates/services/integration/src/show_identifier.rs +++ b/crates/services/integration/src/integration_trait.rs @@ -1,3 +1,7 @@ +use std::future::Future; +use media_models::CommitMediaInput; +use super::{IntegrationMediaCollection, IntegrationMediaSeen}; +use async_graphql::Result as GqlResult; use anyhow::{bail, Result}; use database_models::metadata; use database_utils::ilike_sql; @@ -8,6 +12,25 @@ use sea_orm::{ }; use sea_query::{extension::postgres::PgExpr, Alias, Expr, Func}; +pub trait YankIntegration { + async fn yank_progress( + &self, + ) -> Result<(Vec, Vec)>; +} + +pub trait YankIntegrationWithCommit { + async fn yank_progress( + &self, + commit_metadata: impl Fn(CommitMediaInput) -> F, + ) -> Result<(Vec, Vec)> + where + F: Future>; +} + +pub trait PushIntegration { + async fn push_progress(&self) -> Result<()>; +} + #[async_trait] pub trait ShowIdentifier { fn get_db(&self) -> &DatabaseConnection; @@ -27,7 +50,7 @@ pub trait ShowIdentifier { Expr::col(metadata::Column::ShowSpecifics), Alias::new("text"), )) - .ilike(ilike_sql(episode)), + .ilike(ilike_sql(episode)), ) .add(Expr::col(metadata::Column::Title).ilike(ilike_sql(series))), ) diff --git a/crates/services/integration/src/jellyfin.rs b/crates/services/integration/src/jellyfin.rs index 4bc033aa21..8ceea0af60 100644 --- a/crates/services/integration/src/jellyfin.rs +++ b/crates/services/integration/src/jellyfin.rs @@ -5,7 +5,7 @@ use rust_decimal::Decimal; use rust_decimal_macros::dec; use serde::{Deserialize, Serialize}; -use super::integration::YankIntegration; +use super::integration_trait::YankIntegration; mod models { use super::*; @@ -47,7 +47,7 @@ mod models { } } -pub struct JellyfinIntegration { +pub(crate) struct JellyfinIntegration { payload: String, } diff --git a/crates/services/integration/src/kodi.rs b/crates/services/integration/src/kodi.rs index 8ec718b5ee..67a246b69f 100644 --- a/crates/services/integration/src/kodi.rs +++ b/crates/services/integration/src/kodi.rs @@ -2,9 +2,9 @@ use anyhow::bail; use enums::MediaSource; use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; -use super::integration::YankIntegration; +use super::integration_trait::YankIntegration; -pub struct KodiIntegration { +pub(crate) struct KodiIntegration { payload: String, } impl KodiIntegration { diff --git a/crates/services/integration/src/komga.rs b/crates/services/integration/src/komga.rs index 299dc00711..757851a273 100644 --- a/crates/services/integration/src/komga.rs +++ b/crates/services/integration/src/komga.rs @@ -19,7 +19,7 @@ use sea_query::Expr; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use tokio::sync::{mpsc, mpsc::error::TryRecvError, mpsc::UnboundedReceiver}; -use super::{integration::YankIntegration, IntegrationMediaCollection, IntegrationMediaSeen}; +use super::{integration_trait::YankIntegration, IntegrationMediaCollection, IntegrationMediaSeen}; mod komga_book { use super::*; @@ -181,7 +181,7 @@ impl KomgaEventHandler { } } -pub struct KomgaIntegration { +pub(crate) struct KomgaIntegration { base_url: String, username: String, password: String, diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index 61860a3c26..a045094efe 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -1,24 +1,26 @@ +use std::future::Future; + use anyhow::Result; -use media_models::{IntegrationMediaCollection, IntegrationMediaSeen}; +use async_graphql::Result as GqlResult; +use media_models::{CommitMediaInput, IntegrationMediaCollection, IntegrationMediaSeen}; use sea_orm::DatabaseConnection; - +use database_models::metadata; use crate::{ - audiobookshelf::AudiobookshelfIntegration, emby::EmbyIntegration, integration::PushIntegration, - integration::YankIntegration, integration_type::IntegrationType, jellyfin::JellyfinIntegration, + audiobookshelf::AudiobookshelfIntegration, emby::EmbyIntegration, integration_trait::PushIntegration, + integration_trait::YankIntegration, integration_type::IntegrationType, jellyfin::JellyfinIntegration, kodi::KodiIntegration, komga::KomgaIntegration, plex::PlexIntegration, - radarr::RadarrIntegration, sonarr::SonarrIntegration, + radarr::RadarrIntegration, sonarr::SonarrIntegration, integration_trait::YankIntegrationWithCommit }; mod audiobookshelf; mod emby; -mod integration; +mod integration_trait; pub mod integration_type; mod jellyfin; mod kodi; mod komga; mod plex; mod radarr; -mod show_identifier; mod sonarr; #[derive(Debug)] @@ -64,7 +66,7 @@ impl IntegrationService { ); radarr.push_progress().await } - _ => Ok(()), + _ => Err(anyhow::anyhow!("Unsupported integration type")), } } @@ -90,16 +92,29 @@ impl IntegrationService { let plex = PlexIntegration::new(payload, plex_user, self.db.clone()); plex.yank_progress().await } - IntegrationType::Audiobookshelf(base_url, access_token, isbn_service) => { - let audiobookshelf = - AudiobookshelfIntegration::new(base_url, access_token, isbn_service); - audiobookshelf.yank_progress().await - } IntegrationType::Kodi(payload) => { let kodi = KodiIntegration::new(payload); kodi.yank_progress().await } - _ => Ok((vec![], vec![])), + _ => Err(anyhow::anyhow!("Unsupported integration type")), + } + } + + pub async fn process_progress_commit( + &self, + integration_type: IntegrationType, + commit_metadata: impl Fn(CommitMediaInput) -> F, + ) -> Result<(Vec, Vec)> + where + F: Future> + { + match integration_type { + IntegrationType::Audiobookshelf(base_url, access_token, isbn_service) => { + let audiobookshelf = + AudiobookshelfIntegration::new(base_url, access_token, isbn_service); + audiobookshelf.yank_progress(commit_metadata).await + } + _ => Err(anyhow::anyhow!("Unsupported integration type")), } } } diff --git a/crates/services/integration/src/plex.rs b/crates/services/integration/src/plex.rs index 65de711f13..c1cca8e045 100644 --- a/crates/services/integration/src/plex.rs +++ b/crates/services/integration/src/plex.rs @@ -8,7 +8,7 @@ use rust_decimal_macros::dec; use sea_orm::DatabaseConnection; use serde::{Deserialize, Serialize}; -use super::{integration::YankIntegration, show_identifier::ShowIdentifier}; +use super::{integration_trait::ShowIdentifier, integration_trait::YankIntegration}; mod models { use super::*; @@ -51,7 +51,7 @@ mod models { } } -pub struct PlexIntegration { +pub(crate) struct PlexIntegration { payload: String, plex_user: Option, db: DatabaseConnection, diff --git a/crates/services/integration/src/radarr.rs b/crates/services/integration/src/radarr.rs index 3dc0aa7873..8cd0f1f25a 100644 --- a/crates/services/integration/src/radarr.rs +++ b/crates/services/integration/src/radarr.rs @@ -7,9 +7,9 @@ use radarr_api_rs::{ }; use traits::TraceOk; -use super::integration::PushIntegration; +use super::integration_trait::PushIntegration; -pub struct RadarrIntegration { +pub(crate) struct RadarrIntegration { radarr_base_url: String, radarr_api_key: String, radarr_profile_id: i32, @@ -34,7 +34,7 @@ impl RadarrIntegration { } } - pub async fn radarr_push(&self) -> anyhow::Result<()> { + async fn radarr_push(&self) -> anyhow::Result<()> { let mut configuration = RadarrConfiguration::new(); configuration.base_path = self.radarr_base_url.clone(); configuration.api_key = Some(RadarrApiKey { diff --git a/crates/services/integration/src/sonarr.rs b/crates/services/integration/src/sonarr.rs index 3020996984..261f686dd0 100644 --- a/crates/services/integration/src/sonarr.rs +++ b/crates/services/integration/src/sonarr.rs @@ -7,9 +7,9 @@ use sonarr_api_rs::{ }; use traits::TraceOk; -use super::integration::PushIntegration; +use super::integration_trait::PushIntegration; -pub struct SonarrIntegration { +pub(crate) struct SonarrIntegration { sonarr_base_url: String, sonarr_api_key: String, sonarr_profile_id: i32, diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index 748255d856..48d5157c19 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -4435,11 +4435,13 @@ impl MiscellaneousService { IntegrationProvider::Audiobookshelf => { let specifics = integration.clone().provider_specifics.unwrap(); integration_service - .process_progress(IntegrationType::Audiobookshelf( + .process_progress_commit(IntegrationType::Audiobookshelf( specifics.audiobookshelf_base_url.unwrap(), specifics.audiobookshelf_token.unwrap(), self.get_isbn_service().await.unwrap(), - )) + ), + |input| self.commit_metadata(input), + ) .await } IntegrationProvider::Komga => { From 3eebdb48461b35a258a4c65a98ce7b467d664187 Mon Sep 17 00:00:00 2001 From: jacob-tate Date: Sat, 17 Aug 2024 23:10:05 -0700 Subject: [PATCH 60/61] Undid accidental change --- crates/migrations/src/m20230505_create_review.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/migrations/src/m20230505_create_review.rs b/crates/migrations/src/m20230505_create_review.rs index 893459974a..55d51134f3 100644 --- a/crates/migrations/src/m20230505_create_review.rs +++ b/crates/migrations/src/m20230505_create_review.rs @@ -1,5 +1,5 @@ -use enums::Visibility; use sea_orm_migration::prelude::*; +use enums::Visibility; use super::{ m20230410_create_metadata::Metadata, m20230413_create_person::Person, From ec018ed1c019edbfbb6d05cc3ec37050371efd70 Mon Sep 17 00:00:00 2001 From: Diptesh Choudhuri Date: Mon, 19 Aug 2024 15:50:00 +0530 Subject: [PATCH 61/61] chore(backend): apply formatter --- .../migrations/src/m20230505_create_review.rs | 2 +- .../services/integration/src/audiobookshelf.rs | 14 ++++++-------- .../integration/src/integration_trait.rs | 8 ++++---- crates/services/integration/src/lib.rs | 17 +++++++++-------- crates/services/miscellaneous/src/lib.rs | 13 +++++++------ 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/crates/migrations/src/m20230505_create_review.rs b/crates/migrations/src/m20230505_create_review.rs index 55d51134f3..893459974a 100644 --- a/crates/migrations/src/m20230505_create_review.rs +++ b/crates/migrations/src/m20230505_create_review.rs @@ -1,5 +1,5 @@ -use sea_orm_migration::prelude::*; use enums::Visibility; +use sea_orm_migration::prelude::*; use super::{ m20230410_create_metadata::Metadata, m20230413_create_person::Person, diff --git a/crates/services/integration/src/audiobookshelf.rs b/crates/services/integration/src/audiobookshelf.rs index 963dba6daa..0950767b1e 100644 --- a/crates/services/integration/src/audiobookshelf.rs +++ b/crates/services/integration/src/audiobookshelf.rs @@ -35,7 +35,7 @@ impl YankIntegrationWithCommit for AudiobookshelfIntegration { commit_metadata: impl Fn(CommitMediaInput) -> F, ) -> anyhow::Result<(Vec, Vec)> where - F: Future> + F: Future>, { let client = get_base_http_client( &format!("{}/api/", self.base_url), @@ -89,8 +89,7 @@ impl YankIntegrationWithCommit for AudiobookshelfIntegration { MediaSource::Audible, None, ) - } - else if let Some(itunes_id) = metadata.itunes_id.clone() { + } else if let Some(itunes_id) = metadata.itunes_id.clone() { match &item.recent_episode { Some(pe) => { let lot = MediaLot::Podcast; @@ -116,9 +115,9 @@ impl YankIntegrationWithCommit for AudiobookshelfIntegration { ), _ => { tracing::debug!( - "No podcast found for iTunes ID {:#?}", - itunes_id - ); + "No podcast found for iTunes ID {:#?}", + itunes_id + ); continue; } } @@ -128,8 +127,7 @@ impl YankIntegrationWithCommit for AudiobookshelfIntegration { continue; } } - } - else { + } else { tracing::debug!("No ASIN, ISBN or iTunes ID found for item {:#?}", item); continue; }; diff --git a/crates/services/integration/src/integration_trait.rs b/crates/services/integration/src/integration_trait.rs index a2b17df7a9..3460915c21 100644 --- a/crates/services/integration/src/integration_trait.rs +++ b/crates/services/integration/src/integration_trait.rs @@ -1,16 +1,16 @@ -use std::future::Future; -use media_models::CommitMediaInput; use super::{IntegrationMediaCollection, IntegrationMediaSeen}; -use async_graphql::Result as GqlResult; use anyhow::{bail, Result}; +use async_graphql::Result as GqlResult; use database_models::metadata; use database_utils::ilike_sql; use enums::{MediaLot, MediaSource}; +use media_models::CommitMediaInput; use sea_orm::{ prelude::async_trait::async_trait, ColumnTrait, Condition, DatabaseConnection, EntityTrait, QueryFilter, }; use sea_query::{extension::postgres::PgExpr, Alias, Expr, Func}; +use std::future::Future; pub trait YankIntegration { async fn yank_progress( @@ -50,7 +50,7 @@ pub trait ShowIdentifier { Expr::col(metadata::Column::ShowSpecifics), Alias::new("text"), )) - .ilike(ilike_sql(episode)), + .ilike(ilike_sql(episode)), ) .add(Expr::col(metadata::Column::Title).ilike(ilike_sql(series))), ) diff --git a/crates/services/integration/src/lib.rs b/crates/services/integration/src/lib.rs index a045094efe..5fc68e9426 100644 --- a/crates/services/integration/src/lib.rs +++ b/crates/services/integration/src/lib.rs @@ -1,16 +1,17 @@ use std::future::Future; +use crate::{ + audiobookshelf::AudiobookshelfIntegration, emby::EmbyIntegration, + integration_trait::PushIntegration, integration_trait::YankIntegration, + integration_trait::YankIntegrationWithCommit, integration_type::IntegrationType, + jellyfin::JellyfinIntegration, kodi::KodiIntegration, komga::KomgaIntegration, + plex::PlexIntegration, radarr::RadarrIntegration, sonarr::SonarrIntegration, +}; use anyhow::Result; use async_graphql::Result as GqlResult; +use database_models::metadata; use media_models::{CommitMediaInput, IntegrationMediaCollection, IntegrationMediaSeen}; use sea_orm::DatabaseConnection; -use database_models::metadata; -use crate::{ - audiobookshelf::AudiobookshelfIntegration, emby::EmbyIntegration, integration_trait::PushIntegration, - integration_trait::YankIntegration, integration_type::IntegrationType, jellyfin::JellyfinIntegration, - kodi::KodiIntegration, komga::KomgaIntegration, plex::PlexIntegration, - radarr::RadarrIntegration, sonarr::SonarrIntegration, integration_trait::YankIntegrationWithCommit -}; mod audiobookshelf; mod emby; @@ -106,7 +107,7 @@ impl IntegrationService { commit_metadata: impl Fn(CommitMediaInput) -> F, ) -> Result<(Vec, Vec)> where - F: Future> + F: Future>, { match integration_type { IntegrationType::Audiobookshelf(base_url, access_token, isbn_service) => { diff --git a/crates/services/miscellaneous/src/lib.rs b/crates/services/miscellaneous/src/lib.rs index 46e1288f3d..981d6c2197 100644 --- a/crates/services/miscellaneous/src/lib.rs +++ b/crates/services/miscellaneous/src/lib.rs @@ -4442,12 +4442,13 @@ impl MiscellaneousService { IntegrationProvider::Audiobookshelf => { let specifics = integration.clone().provider_specifics.unwrap(); integration_service - .process_progress_commit(IntegrationType::Audiobookshelf( - specifics.audiobookshelf_base_url.unwrap(), - specifics.audiobookshelf_token.unwrap(), - self.get_isbn_service().await.unwrap(), - ), - |input| self.commit_metadata(input), + .process_progress_commit( + IntegrationType::Audiobookshelf( + specifics.audiobookshelf_base_url.unwrap(), + specifics.audiobookshelf_token.unwrap(), + self.get_isbn_service().await.unwrap(), + ), + |input| self.commit_metadata(input), ) .await }