From 02547f5fc62a55392ed295752a5883f9b8180156 Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Mon, 27 Jan 2025 13:31:09 +0100 Subject: [PATCH] feat: stream translation titles in feed and post page (#4098) --- packages/extension/__tests__/setup.ts | 9 + packages/extension/package.json | 1 + packages/shared/__tests__/setup.ts | 9 + packages/shared/package.json | 1 + packages/shared/src/graphql/feed.ts | 3 + packages/shared/src/graphql/fragments.ts | 6 + packages/shared/src/graphql/posts.ts | 1 + .../shared/src/hooks/post/useSmartTitle.ts | 2 +- .../src/hooks/translation/useTranslation.ts | 166 ++++++++++++++++++ packages/shared/src/hooks/useFeed.ts | 12 +- packages/shared/src/hooks/usePostById.ts | 14 +- packages/shared/src/lib/query.ts | 9 + packages/webapp/__tests__/setup.ts | 9 + packages/webapp/package.json | 1 + pnpm-lock.yaml | 14 ++ 15 files changed, 254 insertions(+), 3 deletions(-) create mode 100644 packages/shared/src/hooks/translation/useTranslation.ts diff --git a/packages/extension/__tests__/setup.ts b/packages/extension/__tests__/setup.ts index 12cf7c8fdf..25dbd9656b 100644 --- a/packages/extension/__tests__/setup.ts +++ b/packages/extension/__tests__/setup.ts @@ -71,4 +71,13 @@ Object.defineProperty(global, 'open', { value: jest.fn(), }); +Object.defineProperty(global, 'TransformStream', { + writable: true, + value: jest.fn().mockImplementation(() => ({ + backpressure: jest.fn(), + readable: jest.fn(), + writable: jest.fn(), + })), +}); + structuredCloneJsonPolyfill(); diff --git a/packages/extension/package.json b/packages/extension/package.json index 69612c4e2e..b16386a069 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -24,6 +24,7 @@ "date-fns": "^2.25.0", "date-fns-tz": "1.2.2", "dompurify": "^2.5.4", + "fetch-event-stream": "^0.1.5", "focus-visible": "^5.2.1", "graphql": "^16.9.0", "graphql-request": "^3.6.1", diff --git a/packages/shared/__tests__/setup.ts b/packages/shared/__tests__/setup.ts index b79ec45a7a..40089ae541 100644 --- a/packages/shared/__tests__/setup.ts +++ b/packages/shared/__tests__/setup.ts @@ -53,6 +53,15 @@ Object.defineProperty(global, 'open', { value: jest.fn(), }); +Object.defineProperty(global, 'TransformStream', { + writable: true, + value: jest.fn().mockImplementation(() => ({ + backpressure: jest.fn(), + readable: jest.fn(), + writable: jest.fn(), + })), +}); + jest.mock('next/router', () => ({ useRouter: jest.fn().mockImplementation( () => diff --git a/packages/shared/package.json b/packages/shared/package.json index 5bc72dceb6..e1205f98c8 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -106,6 +106,7 @@ "@paddle/paddle-js": "^1.3.2", "@tippyjs/react": "^4.2.6", "check-password-strength": "^2.0.10", + "fetch-event-stream": "^0.1.5", "graphql-ws": "^5.5.5", "node-fetch": "^2.6.6", "react-markdown": "^8.0.7", diff --git a/packages/shared/src/graphql/feed.ts b/packages/shared/src/graphql/feed.ts index 9227ec808a..f48078bf7d 100644 --- a/packages/shared/src/graphql/feed.ts +++ b/packages/shared/src/graphql/feed.ts @@ -79,6 +79,9 @@ export const FEED_POST_FRAGMENT = gql` } slug clickbaitTitleDetected + translation { + title + } } trending feedMeta diff --git a/packages/shared/src/graphql/fragments.ts b/packages/shared/src/graphql/fragments.ts index 5f66d4319b..c1e19c8b96 100644 --- a/packages/shared/src/graphql/fragments.ts +++ b/packages/shared/src/graphql/fragments.ts @@ -219,6 +219,9 @@ export const FEED_POST_INFO_FRAGMENT = gql` } slug clickbaitTitleDetected + translation { + title + } } `; @@ -269,6 +272,9 @@ export const SHARED_POST_INFO_FRAGMENT = gql` slug domain clickbaitTitleDetected + translation { + title + } } ${PRIVILEGED_MEMBERS_FRAGMENT} ${SOURCE_BASE_FRAGMENT} diff --git a/packages/shared/src/graphql/posts.ts b/packages/shared/src/graphql/posts.ts index fe63312c02..e147604983 100644 --- a/packages/shared/src/graphql/posts.ts +++ b/packages/shared/src/graphql/posts.ts @@ -129,6 +129,7 @@ export interface Post { bookmarkList?: BookmarkFolder; domain?: string; clickbaitTitleDetected?: boolean; + translation?: { title?: boolean }; } export type RelatedPost = Pick< diff --git a/packages/shared/src/hooks/post/useSmartTitle.ts b/packages/shared/src/hooks/post/useSmartTitle.ts index 2eeb9588ea..1b8ddc7adc 100644 --- a/packages/shared/src/hooks/post/useSmartTitle.ts +++ b/packages/shared/src/hooks/post/useSmartTitle.ts @@ -114,7 +114,7 @@ export const useSmartTitle = (post: Post): UseSmartTitle => { return fetchedSmartTitle ? smartTitle : post?.title || post?.sharedPost?.title; - }, [fetchedSmartTitle, smartTitle, post]); + }, [fetchedSmartTitle, smartTitle, post?.title, post?.sharedPost?.title]); const shieldActive = useMemo(() => { return ( diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts new file mode 100644 index 0000000000..231cfcb635 --- /dev/null +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -0,0 +1,166 @@ +import { useCallback, useEffect, useRef } from 'react'; +import type { InfiniteData, QueryKey } from '@tanstack/react-query'; +import { useQueryClient } from '@tanstack/react-query'; +import { events } from 'fetch-event-stream'; +import { useAuthContext } from '../../contexts/AuthContext'; +import { apiUrl } from '../../lib/config'; +import type { FeedData, Post } from '../../graphql/posts'; +import { + updateCachedPagePost, + findIndexOfPostInData, + updatePostCache, +} from '../../lib/query'; +import { useSettingsContext } from '../../contexts/SettingsContext'; + +export enum ServerEvents { + Connect = 'connect', + Message = 'message', + Disconnect = 'disconnect', + Error = 'error', +} + +type UseTranslation = (props: { + queryKey: QueryKey; + queryType: 'post' | 'feed'; +}) => { + fetchTranslations: (id: Post[]) => void; +}; + +type TranslateEvent = { + id: string; + title: string; +}; + +const updateTranslation = (post: Post, translation: TranslateEvent): Post => { + const updatedPost = post; + if (post.title) { + updatedPost.title = translation.title; + updatedPost.translation = { title: !!translation.title }; + } else { + updatedPost.sharedPost.title = translation.title; + updatedPost.sharedPost.translation = { title: !!translation.title }; + } + + return updatedPost; +}; + +export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { + const abort = useRef(); + const { user, accessToken, isLoggedIn } = useAuthContext(); + const { flags } = useSettingsContext(); + const queryClient = useQueryClient(); + + const { language } = user || {}; + const isStreamActive = isLoggedIn && !!language; + + const updateFeed = useCallback( + (translatedPost: TranslateEvent) => { + const updatePost = updateCachedPagePost(queryKey, queryClient); + const feedData = + queryClient.getQueryData>(queryKey); + const { pageIndex, index } = findIndexOfPostInData( + feedData, + translatedPost.id, + true, + ); + if (index > -1) { + updatePost( + pageIndex, + index, + updateTranslation( + feedData.pages[pageIndex].page.edges[index].node, + translatedPost, + ), + ); + } + }, + [queryKey, queryClient], + ); + + const updatePost = useCallback( + (translatedPost: TranslateEvent) => { + updatePostCache(queryClient, translatedPost.id, (post) => + updateTranslation(post, translatedPost), + ); + }, + [queryClient], + ); + + const fetchTranslations = useCallback( + async (posts: Post[]) => { + if (!isStreamActive) { + return; + } + if (posts.length === 0) { + return; + } + + const postIds = posts + .filter((node) => + node?.title + ? !node?.translation?.title + : !node?.sharedPost?.translation?.title, + ) + .filter((node) => + flags.clickbaitShieldEnabled && node?.title + ? !node.clickbaitTitleDetected + : !node.sharedPost?.clickbaitTitleDetected, + ) + .filter(Boolean) + .map((node) => (node?.title ? node.id : node?.sharedPost.id)); + + if (postIds.length === 0) { + return; + } + + const params = new URLSearchParams(); + postIds.forEach((id) => { + params.append('id', id); + }); + + const response = await fetch(`${apiUrl}/translate/post/title?${params}`, { + signal: abort.current?.signal, + headers: { + Accept: 'text/event-stream', + Authorization: `Bearer ${accessToken?.token}`, + 'Content-Language': language as string, + }, + }); + + if (!response.ok) { + return; + } + + // eslint-disable-next-line no-restricted-syntax + for await (const message of events(response)) { + if (message.event === ServerEvents.Message) { + const post = JSON.parse(message.data) as TranslateEvent; + if (queryType === 'feed') { + updateFeed(post); + } else { + updatePost(post); + } + } + } + }, + [ + accessToken?.token, + flags.clickbaitShieldEnabled, + isStreamActive, + language, + queryType, + updateFeed, + updatePost, + ], + ); + + useEffect(() => { + abort.current = new AbortController(); + + return () => { + abort.current?.abort(); + }; + }, []); + + return { fetchTranslations }; +}; diff --git a/packages/shared/src/hooks/useFeed.ts b/packages/shared/src/hooks/useFeed.ts index 1a00c2c040..a255c8427c 100644 --- a/packages/shared/src/hooks/useFeed.ts +++ b/packages/shared/src/hooks/useFeed.ts @@ -26,9 +26,11 @@ import { featureFeedAdTemplate } from '../lib/featureManagement'; import { cloudinaryPostImageCoverPlaceholder } from '../lib/image'; import { AD_PLACEHOLDER_SOURCE_ID } from '../lib/constants'; import { SharedFeedPage } from '../components/utilities'; +import { useTranslation } from './translation/useTranslation'; interface FeedItemBase { type: T; + dataUpdatedAt: number; } interface AdItem extends FeedItemBase { @@ -101,6 +103,10 @@ export default function useFeed( const { user, tokenRefreshed } = useContext(AuthContext); const { isPlus } = usePlusSubscription(); const queryClient = useQueryClient(); + const { fetchTranslations } = useTranslation({ + queryKey: feedQueryKey, + queryType: 'feed', + }); const isFeedPreview = feedQueryKey?.[0] === RequestKey.FeedPreview; const avoidRetry = params?.settings?.feedName === SharedFeedPage.Custom && !isPlus; @@ -108,7 +114,7 @@ export default function useFeed( const feedQuery = useInfiniteQuery({ queryKey: feedQueryKey, queryFn: async ({ pageParam }) => { - const res = await gqlClient.request(query, { + const res = await gqlClient.request(query, { ...variables, first: pageSize, after: pageParam, @@ -132,6 +138,8 @@ export default function useFeed( } } + fetchTranslations(res.page.edges.map(({ node }) => node)); + return res; }, refetchOnMount: false, @@ -287,6 +295,7 @@ export default function useFeed( post: node, page: pageIndex, index, + dataUpdatedAt: feedQuery.dataUpdatedAt, }); }); @@ -302,6 +311,7 @@ export default function useFeed( }, [ feedQuery.data, feedQuery.isFetching, + feedQuery.dataUpdatedAt, settings.marketingCta, settings.showAcquisitionForm, placeholdersPerPage, diff --git a/packages/shared/src/hooks/usePostById.ts b/packages/shared/src/hooks/usePostById.ts index f6aff131dd..fe65def3a7 100644 --- a/packages/shared/src/hooks/usePostById.ts +++ b/packages/shared/src/hooks/usePostById.ts @@ -26,6 +26,7 @@ import { mutationKeyToContentPreferenceStatusMap, } from './contentPreference/types'; import type { PropsParameters } from '../types'; +import { useTranslation } from './translation/useTranslation'; interface UsePostByIdProps { id: string; @@ -94,13 +95,24 @@ const usePostById = ({ id, options = {} }: UsePostByIdProps): UsePostById => { const { initialData, ...restOptions } = options; const { tokenRefreshed } = useAuthContext(); const key = getPostByIdKey(id); + const { fetchTranslations } = useTranslation({ + queryKey: key, + queryType: 'post', + }); const { data: postById, isError, isPending, } = useQuery({ queryKey: key, - queryFn: () => gqlClient.request(POST_BY_ID_QUERY, { id }), + queryFn: async () => { + const res = await gqlClient.request(POST_BY_ID_QUERY, { id }); + if (!res.post?.translation?.title) { + fetchTranslations([res.post]); + } + + return res; + }, ...restOptions, staleTime: StaleTime.Default, enabled: !!id && tokenRefreshed, diff --git a/packages/shared/src/lib/query.ts b/packages/shared/src/lib/query.ts index 19bd6f4cfa..93db70870a 100644 --- a/packages/shared/src/lib/query.ts +++ b/packages/shared/src/lib/query.ts @@ -11,6 +11,7 @@ import { GARMR_ERROR } from '../graphql/common'; import type { PageInfo, Connection } from '../graphql/common'; import type { EmptyObjectLiteral } from './kratos'; import type { LoggedUser } from './user'; +import { PostType } from '../graphql/posts'; import type { FeedData, Post, @@ -482,6 +483,7 @@ export const getAllCommentsQuery = (postId: string): QueryKeyReturnType[] => { export const findIndexOfPostInData = ( data: InfiniteData, id: string, + findBySharedPost = false, ): { pageIndex: number; index: number } => { for (let pageIndex = 0; pageIndex < data.pages.length; pageIndex += 1) { const page = data.pages[pageIndex]; @@ -490,6 +492,13 @@ export const findIndexOfPostInData = ( if (item.node.id === id) { return { pageIndex, index }; } + if ( + findBySharedPost && + item.node.type === PostType.Share && + item.node.sharedPost.id === id + ) { + return { pageIndex, index }; + } } } return { pageIndex: -1, index: -1 }; diff --git a/packages/webapp/__tests__/setup.ts b/packages/webapp/__tests__/setup.ts index 6074bf1fab..40a855c517 100644 --- a/packages/webapp/__tests__/setup.ts +++ b/packages/webapp/__tests__/setup.ts @@ -38,6 +38,15 @@ Object.defineProperty(global, 'open', { value: jest.fn(), }); +Object.defineProperty(global, 'TransformStream', { + writable: true, + value: jest.fn().mockImplementation(() => ({ + backpressure: jest.fn(), + readable: jest.fn(), + writable: jest.fn(), + })), +}); + jest.mock('next/router', () => ({ useRouter: jest.fn().mockImplementation( () => diff --git a/packages/webapp/package.json b/packages/webapp/package.json index f517f4ef84..46e0d79413 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -24,6 +24,7 @@ "date-fns": "^2.28.0", "date-fns-tz": "1.2.2", "dompurify": "^2.5.4", + "fetch-event-stream": "^0.1.5", "focus-visible": "^5.2.1", "graphql": "^16.9.0", "graphql-request": "^3.7.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32b325621c..5f40b6460f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -106,6 +106,9 @@ importers: dompurify: specifier: ^2.5.4 version: 2.5.7 + fetch-event-stream: + specifier: ^0.1.5 + version: 0.1.5 focus-visible: specifier: ^5.2.1 version: 5.2.1 @@ -394,6 +397,9 @@ importers: classnames: specifier: ^2.3.1 version: 2.5.1 + fetch-event-stream: + specifier: ^0.1.5 + version: 0.1.5 graphql-ws: specifier: ^5.5.5 version: 5.16.0(graphql@16.9.0) @@ -810,6 +816,9 @@ importers: dompurify: specifier: ^2.5.4 version: 2.5.7 + fetch-event-stream: + specifier: ^0.1.5 + version: 0.1.5 focus-visible: specifier: ^5.2.1 version: 5.2.1 @@ -5021,6 +5030,9 @@ packages: picomatch: optional: true + fetch-event-stream@0.1.5: + resolution: {integrity: sha512-V1PWovkspxQfssq/NnxoEyQo1DV+MRK/laPuPblIZmSjMN8P5u46OhlFQznSr9p/t0Sp8Uc6SbM3yCMfr0KU8g==} + file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -13373,6 +13385,8 @@ snapshots: optionalDependencies: picomatch: 4.0.2 + fetch-event-stream@0.1.5: {} + file-entry-cache@6.0.1: dependencies: flat-cache: 3.2.0