From 2b5b7c97aa7512ddb3c8125fa934eb89d8163080 Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Thu, 23 Jan 2025 07:06:05 +0100 Subject: [PATCH 01/15] feat: stream translation titles in feed --- packages/extension/package.json | 1 + packages/shared/package.json | 1 + packages/shared/src/graphql/feed.ts | 3 + packages/shared/src/graphql/fragments.ts | 3 + packages/shared/src/graphql/posts.ts | 1 + .../src/hooks/translation/useTranslation.ts | 105 ++++++++++++++++++ packages/shared/src/hooks/useFeed.ts | 10 +- packages/shared/src/lib/query.ts | 9 ++ packages/webapp/package.json | 1 + pnpm-lock.yaml | 14 +++ 10 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 packages/shared/src/hooks/translation/useTranslation.ts diff --git a/packages/extension/package.json b/packages/extension/package.json index 69612c4e2e..b16386a069 100644 --- a/packages/extension/package.json +++ b/packages/extension/package.json @@ -24,6 +24,7 @@ "date-fns": "^2.25.0", "date-fns-tz": "1.2.2", "dompurify": "^2.5.4", + "fetch-event-stream": "^0.1.5", "focus-visible": "^5.2.1", "graphql": "^16.9.0", "graphql-request": "^3.6.1", diff --git a/packages/shared/package.json b/packages/shared/package.json index 5bc72dceb6..e1205f98c8 100644 --- a/packages/shared/package.json +++ b/packages/shared/package.json @@ -106,6 +106,7 @@ "@paddle/paddle-js": "^1.3.2", "@tippyjs/react": "^4.2.6", "check-password-strength": "^2.0.10", + "fetch-event-stream": "^0.1.5", "graphql-ws": "^5.5.5", "node-fetch": "^2.6.6", "react-markdown": "^8.0.7", diff --git a/packages/shared/src/graphql/feed.ts b/packages/shared/src/graphql/feed.ts index 9227ec808a..f48078bf7d 100644 --- a/packages/shared/src/graphql/feed.ts +++ b/packages/shared/src/graphql/feed.ts @@ -79,6 +79,9 @@ export const FEED_POST_FRAGMENT = gql` } slug clickbaitTitleDetected + translation { + title + } } trending feedMeta diff --git a/packages/shared/src/graphql/fragments.ts b/packages/shared/src/graphql/fragments.ts index 5f66d4319b..ba4bfab0a8 100644 --- a/packages/shared/src/graphql/fragments.ts +++ b/packages/shared/src/graphql/fragments.ts @@ -219,6 +219,9 @@ export const FEED_POST_INFO_FRAGMENT = gql` } slug clickbaitTitleDetected + translation { + title + } } `; diff --git a/packages/shared/src/graphql/posts.ts b/packages/shared/src/graphql/posts.ts index fe63312c02..e147604983 100644 --- a/packages/shared/src/graphql/posts.ts +++ b/packages/shared/src/graphql/posts.ts @@ -129,6 +129,7 @@ export interface Post { bookmarkList?: BookmarkFolder; domain?: string; clickbaitTitleDetected?: boolean; + translation?: { title?: boolean }; } export type RelatedPost = Pick< diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts new file mode 100644 index 0000000000..8a272aa696 --- /dev/null +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -0,0 +1,105 @@ +import { useCallback, useEffect, useRef } from 'react'; +import type { InfiniteData, QueryKey } from '@tanstack/react-query'; +import { useQueryClient } from '@tanstack/react-query'; +import { stream } from 'fetch-event-stream'; +import { useAuthContext } from '../../contexts/AuthContext'; +import { apiUrl } from '../../lib/config'; +import type { FeedData } from '../../graphql/posts'; +import { updateCachedPagePost, findIndexOfPostInData } from '../../lib/query'; +import type { LoggedUser } from '../../lib/user'; + +export enum ServerEvents { + Connect = 'connect', + Message = 'message', + Disconnect = 'disconnect', + Error = 'error', +} + +type UseTranslation = (feedQueryKey: QueryKey) => { + fetchTranslations: (id: string[]) => void; +}; + +type TranslateEvent = { + id: string; + title: string; +}; + +export const useTranslation: UseTranslation = (feedQueryKey) => { + const abort = useRef(); + const { user, accessToken, isLoggedIn } = useAuthContext(); + + const queryClient = useQueryClient(); + + const updatePostTranslation = useCallback( + (post: TranslateEvent) => { + const updatePost = updateCachedPagePost(feedQueryKey, queryClient); + const feedData = + queryClient.getQueryData>(feedQueryKey); + const { pageIndex, index } = findIndexOfPostInData( + feedData, + post.id, + true, + ); + if (index > -1) { + const updatedPost = feedData.pages[pageIndex].page.edges[index].node; + if (updatedPost.title) { + updatedPost.title = post.title; + updatedPost.translation = { title: !!post.title }; + } else { + updatedPost.sharedPost.title = post.title; + updatedPost.sharedPost.translation = { + title: !!post.title, + }; + } + + updatePost(pageIndex, index, updatedPost); + } + }, + [feedQueryKey, queryClient], + ); + + const fetchTranslations = useCallback( + async (postIds: string[]) => { + if (!isLoggedIn) { + return; + } + if (postIds.length === 0) { + return; + } + + const params = new URLSearchParams(); + postIds.forEach((id) => { + params.append('id', id); + }); + + const messages = await stream( + `${apiUrl}/translate/post/title?${params}`, + { + headers: { + Authorization: `Bearer ${accessToken?.token}`, + 'Content-Language': (user as LoggedUser).language as string, + }, + }, + ); + + // eslint-disable-next-line no-restricted-syntax + for await (const message of messages) { + if (message.event === ServerEvents.Message) { + const post = JSON.parse(message.data) as TranslateEvent; + updatePostTranslation(post); + } + } + }, + [accessToken?.token, isLoggedIn, updatePostTranslation, user], + ); + + useEffect(() => { + abort.current = new AbortController(); + + return () => { + abort.current?.abort(); + }; + }, []); + + return { fetchTranslations }; +}; diff --git a/packages/shared/src/hooks/useFeed.ts b/packages/shared/src/hooks/useFeed.ts index 912aa06829..68a78127ae 100644 --- a/packages/shared/src/hooks/useFeed.ts +++ b/packages/shared/src/hooks/useFeed.ts @@ -26,6 +26,7 @@ import { featureFeedAdTemplate } from '../lib/featureManagement'; import { cloudinaryPostImageCoverPlaceholder } from '../lib/image'; import { AD_PLACEHOLDER_SOURCE_ID } from '../lib/constants'; import { SharedFeedPage } from '../components/utilities'; +import { useTranslation } from './translation/useTranslation'; interface FeedItemBase { type: T; @@ -101,6 +102,7 @@ export default function useFeed( const { user, tokenRefreshed } = useContext(AuthContext); const { showPlusSubscription, isPlus } = usePlusSubscription(); const queryClient = useQueryClient(); + const { fetchTranslations } = useTranslation(feedQueryKey); const isFeedPreview = feedQueryKey?.[0] === RequestKey.FeedPreview; const avoidRetry = params?.settings?.feedName === SharedFeedPage.Custom && @@ -110,7 +112,7 @@ export default function useFeed( const feedQuery = useInfiniteQuery({ queryKey: feedQueryKey, queryFn: async ({ pageParam }) => { - const res = await gqlClient2.request(query, { + const res = await gqlClient2.request(query, { ...variables, first: pageSize, after: pageParam, @@ -134,6 +136,12 @@ export default function useFeed( } } + fetchTranslations( + res.page.edges + .filter(({ node }) => !node?.translation?.title) + .map(({ node }) => (node?.title ? node.id : node?.sharedPost.id)), + ); + return res; }, refetchOnMount: false, diff --git a/packages/shared/src/lib/query.ts b/packages/shared/src/lib/query.ts index 19bd6f4cfa..93db70870a 100644 --- a/packages/shared/src/lib/query.ts +++ b/packages/shared/src/lib/query.ts @@ -11,6 +11,7 @@ import { GARMR_ERROR } from '../graphql/common'; import type { PageInfo, Connection } from '../graphql/common'; import type { EmptyObjectLiteral } from './kratos'; import type { LoggedUser } from './user'; +import { PostType } from '../graphql/posts'; import type { FeedData, Post, @@ -482,6 +483,7 @@ export const getAllCommentsQuery = (postId: string): QueryKeyReturnType[] => { export const findIndexOfPostInData = ( data: InfiniteData, id: string, + findBySharedPost = false, ): { pageIndex: number; index: number } => { for (let pageIndex = 0; pageIndex < data.pages.length; pageIndex += 1) { const page = data.pages[pageIndex]; @@ -490,6 +492,13 @@ export const findIndexOfPostInData = ( if (item.node.id === id) { return { pageIndex, index }; } + if ( + findBySharedPost && + item.node.type === PostType.Share && + item.node.sharedPost.id === id + ) { + return { pageIndex, index }; + } } } return { pageIndex: -1, index: -1 }; diff --git a/packages/webapp/package.json b/packages/webapp/package.json index f517f4ef84..46e0d79413 100644 --- a/packages/webapp/package.json +++ b/packages/webapp/package.json @@ -24,6 +24,7 @@ "date-fns": "^2.28.0", "date-fns-tz": "1.2.2", "dompurify": "^2.5.4", + "fetch-event-stream": "^0.1.5", "focus-visible": "^5.2.1", "graphql": "^16.9.0", "graphql-request": "^3.7.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32b325621c..5f40b6460f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -106,6 +106,9 @@ importers: dompurify: specifier: ^2.5.4 version: 2.5.7 + fetch-event-stream: + specifier: ^0.1.5 + version: 0.1.5 focus-visible: specifier: ^5.2.1 version: 5.2.1 @@ -394,6 +397,9 @@ importers: classnames: specifier: ^2.3.1 version: 2.5.1 + fetch-event-stream: + specifier: ^0.1.5 + version: 0.1.5 graphql-ws: specifier: ^5.5.5 version: 5.16.0(graphql@16.9.0) @@ -810,6 +816,9 @@ importers: dompurify: specifier: ^2.5.4 version: 2.5.7 + fetch-event-stream: + specifier: ^0.1.5 + version: 0.1.5 focus-visible: specifier: ^5.2.1 version: 5.2.1 @@ -5021,6 +5030,9 @@ packages: picomatch: optional: true + fetch-event-stream@0.1.5: + resolution: {integrity: sha512-V1PWovkspxQfssq/NnxoEyQo1DV+MRK/laPuPblIZmSjMN8P5u46OhlFQznSr9p/t0Sp8Uc6SbM3yCMfr0KU8g==} + file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -13373,6 +13385,8 @@ snapshots: optionalDependencies: picomatch: 4.0.2 + fetch-event-stream@0.1.5: {} + file-entry-cache@6.0.1: dependencies: flat-cache: 3.2.0 From cb293c968ece2140c49292964e91cbd81cf72243 Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Thu, 23 Jan 2025 09:06:15 +0100 Subject: [PATCH 02/15] fix: don't call Kvasir if language is unset --- .../shared/src/hooks/translation/useTranslation.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index 8a272aa696..12bf32c283 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -6,7 +6,6 @@ import { useAuthContext } from '../../contexts/AuthContext'; import { apiUrl } from '../../lib/config'; import type { FeedData } from '../../graphql/posts'; import { updateCachedPagePost, findIndexOfPostInData } from '../../lib/query'; -import type { LoggedUser } from '../../lib/user'; export enum ServerEvents { Connect = 'connect', @@ -27,9 +26,10 @@ type TranslateEvent = { export const useTranslation: UseTranslation = (feedQueryKey) => { const abort = useRef(); const { user, accessToken, isLoggedIn } = useAuthContext(); - const queryClient = useQueryClient(); + const { language } = user; + const updatePostTranslation = useCallback( (post: TranslateEvent) => { const updatePost = updateCachedPagePost(feedQueryKey, queryClient); @@ -60,7 +60,7 @@ export const useTranslation: UseTranslation = (feedQueryKey) => { const fetchTranslations = useCallback( async (postIds: string[]) => { - if (!isLoggedIn) { + if (!isLoggedIn || !language) { return; } if (postIds.length === 0) { @@ -77,7 +77,7 @@ export const useTranslation: UseTranslation = (feedQueryKey) => { { headers: { Authorization: `Bearer ${accessToken?.token}`, - 'Content-Language': (user as LoggedUser).language as string, + 'Content-Language': language as string, }, }, ); @@ -90,7 +90,7 @@ export const useTranslation: UseTranslation = (feedQueryKey) => { } } }, - [accessToken?.token, isLoggedIn, updatePostTranslation, user], + [accessToken?.token, isLoggedIn, language, updatePostTranslation], ); useEffect(() => { From cc575fbe67567f9d1ae8a1c0f7cbcf977604121c Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Thu, 23 Jan 2025 09:21:40 +0100 Subject: [PATCH 03/15] fix: prepare to support translation on post page --- .../src/hooks/translation/useTranslation.ts | 21 ++++++++++++------- packages/shared/src/hooks/useFeed.ts | 5 ++++- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index 12bf32c283..2d2b602eb0 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -14,7 +14,10 @@ export enum ServerEvents { Error = 'error', } -type UseTranslation = (feedQueryKey: QueryKey) => { +type UseTranslation = (props: { + queryKey: QueryKey; + queryType: 'post' | 'feed'; +}) => { fetchTranslations: (id: string[]) => void; }; @@ -23,18 +26,18 @@ type TranslateEvent = { title: string; }; -export const useTranslation: UseTranslation = (feedQueryKey) => { +export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { const abort = useRef(); const { user, accessToken, isLoggedIn } = useAuthContext(); const queryClient = useQueryClient(); const { language } = user; - const updatePostTranslation = useCallback( + const updateFeed = useCallback( (post: TranslateEvent) => { - const updatePost = updateCachedPagePost(feedQueryKey, queryClient); + const updatePost = updateCachedPagePost(queryKey, queryClient); const feedData = - queryClient.getQueryData>(feedQueryKey); + queryClient.getQueryData>(queryKey); const { pageIndex, index } = findIndexOfPostInData( feedData, post.id, @@ -55,7 +58,7 @@ export const useTranslation: UseTranslation = (feedQueryKey) => { updatePost(pageIndex, index, updatedPost); } }, - [feedQueryKey, queryClient], + [queryKey, queryClient], ); const fetchTranslations = useCallback( @@ -86,11 +89,13 @@ export const useTranslation: UseTranslation = (feedQueryKey) => { for await (const message of messages) { if (message.event === ServerEvents.Message) { const post = JSON.parse(message.data) as TranslateEvent; - updatePostTranslation(post); + if (queryType === 'feed') { + updateFeed(post); + } } } }, - [accessToken?.token, isLoggedIn, language, updatePostTranslation], + [accessToken?.token, isLoggedIn, language, queryType, updateFeed], ); useEffect(() => { diff --git a/packages/shared/src/hooks/useFeed.ts b/packages/shared/src/hooks/useFeed.ts index 68a78127ae..916c6fb927 100644 --- a/packages/shared/src/hooks/useFeed.ts +++ b/packages/shared/src/hooks/useFeed.ts @@ -102,7 +102,10 @@ export default function useFeed( const { user, tokenRefreshed } = useContext(AuthContext); const { showPlusSubscription, isPlus } = usePlusSubscription(); const queryClient = useQueryClient(); - const { fetchTranslations } = useTranslation(feedQueryKey); + const { fetchTranslations } = useTranslation({ + queryKey: feedQueryKey, + queryType: 'feed', + }); const isFeedPreview = feedQueryKey?.[0] === RequestKey.FeedPreview; const avoidRetry = params?.settings?.feedName === SharedFeedPage.Custom && From b65f69e414f7efd493fdcd264db7b741317deffa Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Thu, 23 Jan 2025 09:56:36 +0100 Subject: [PATCH 04/15] feat: stream translation on post page --- packages/shared/src/graphql/fragments.ts | 3 + .../shared/src/hooks/post/useSmartTitle.ts | 2 +- .../src/hooks/translation/useTranslation.ts | 67 ++++++++++++++----- packages/shared/src/hooks/usePostById.ts | 14 +++- 4 files changed, 66 insertions(+), 20 deletions(-) diff --git a/packages/shared/src/graphql/fragments.ts b/packages/shared/src/graphql/fragments.ts index ba4bfab0a8..c1e19c8b96 100644 --- a/packages/shared/src/graphql/fragments.ts +++ b/packages/shared/src/graphql/fragments.ts @@ -272,6 +272,9 @@ export const SHARED_POST_INFO_FRAGMENT = gql` slug domain clickbaitTitleDetected + translation { + title + } } ${PRIVILEGED_MEMBERS_FRAGMENT} ${SOURCE_BASE_FRAGMENT} diff --git a/packages/shared/src/hooks/post/useSmartTitle.ts b/packages/shared/src/hooks/post/useSmartTitle.ts index c16d08ebfc..dfe5d796c9 100644 --- a/packages/shared/src/hooks/post/useSmartTitle.ts +++ b/packages/shared/src/hooks/post/useSmartTitle.ts @@ -120,7 +120,7 @@ export const useSmartTitle = (post: Post): UseSmartTitle => { return fetchedSmartTitle ? smartTitle : post?.title || post?.sharedPost?.title; - }, [fetchedSmartTitle, smartTitle, post]); + }, [fetchedSmartTitle, smartTitle, post?.title, post?.sharedPost?.title]); const shieldActive = useMemo(() => { return ( diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index 2d2b602eb0..ec12ff3523 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -4,8 +4,12 @@ import { useQueryClient } from '@tanstack/react-query'; import { stream } from 'fetch-event-stream'; import { useAuthContext } from '../../contexts/AuthContext'; import { apiUrl } from '../../lib/config'; -import type { FeedData } from '../../graphql/posts'; -import { updateCachedPagePost, findIndexOfPostInData } from '../../lib/query'; +import type { FeedData, Post } from '../../graphql/posts'; +import { + updateCachedPagePost, + findIndexOfPostInData, + updatePostCache, +} from '../../lib/query'; export enum ServerEvents { Connect = 'connect', @@ -26,41 +30,59 @@ type TranslateEvent = { title: string; }; +const updateTranslation = (post: Post, translation: TranslateEvent): Post => { + const updatedPost = post; + if (post.title) { + updatedPost.title = translation.title; + updatedPost.translation = { title: !!translation.title }; + } else { + updatedPost.sharedPost.title = translation.title; + updatedPost.sharedPost.translation = { title: !!translation.title }; + } + + return updatedPost; +}; + export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { const abort = useRef(); const { user, accessToken, isLoggedIn } = useAuthContext(); const queryClient = useQueryClient(); - const { language } = user; + const { language } = user || {}; const updateFeed = useCallback( - (post: TranslateEvent) => { + (translatedPost: TranslateEvent) => { const updatePost = updateCachedPagePost(queryKey, queryClient); const feedData = queryClient.getQueryData>(queryKey); const { pageIndex, index } = findIndexOfPostInData( feedData, - post.id, + translatedPost.id, true, ); if (index > -1) { - const updatedPost = feedData.pages[pageIndex].page.edges[index].node; - if (updatedPost.title) { - updatedPost.title = post.title; - updatedPost.translation = { title: !!post.title }; - } else { - updatedPost.sharedPost.title = post.title; - updatedPost.sharedPost.translation = { - title: !!post.title, - }; - } - - updatePost(pageIndex, index, updatedPost); + updatePost( + pageIndex, + index, + updateTranslation( + feedData.pages[pageIndex].page.edges[index].node, + translatedPost, + ), + ); } }, [queryKey, queryClient], ); + const updatePost = useCallback( + (translatedPost: TranslateEvent) => { + updatePostCache(queryClient, translatedPost.id, (post) => + updateTranslation(post, translatedPost), + ); + }, + [queryClient], + ); + const fetchTranslations = useCallback( async (postIds: string[]) => { if (!isLoggedIn || !language) { @@ -91,11 +113,20 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { const post = JSON.parse(message.data) as TranslateEvent; if (queryType === 'feed') { updateFeed(post); + } else { + updatePost(post); } } } }, - [accessToken?.token, isLoggedIn, language, queryType, updateFeed], + [ + accessToken?.token, + isLoggedIn, + language, + queryType, + updateFeed, + updatePost, + ], ); useEffect(() => { diff --git a/packages/shared/src/hooks/usePostById.ts b/packages/shared/src/hooks/usePostById.ts index f6aff131dd..4190a68ab7 100644 --- a/packages/shared/src/hooks/usePostById.ts +++ b/packages/shared/src/hooks/usePostById.ts @@ -26,6 +26,7 @@ import { mutationKeyToContentPreferenceStatusMap, } from './contentPreference/types'; import type { PropsParameters } from '../types'; +import { useTranslation } from './translation/useTranslation'; interface UsePostByIdProps { id: string; @@ -94,13 +95,24 @@ const usePostById = ({ id, options = {} }: UsePostByIdProps): UsePostById => { const { initialData, ...restOptions } = options; const { tokenRefreshed } = useAuthContext(); const key = getPostByIdKey(id); + const { fetchTranslations } = useTranslation({ + queryKey: key, + queryType: 'post', + }); const { data: postById, isError, isPending, } = useQuery({ queryKey: key, - queryFn: () => gqlClient.request(POST_BY_ID_QUERY, { id }), + queryFn: async () => { + const res = await gqlClient.request(POST_BY_ID_QUERY, { id }); + if (!res.post?.translation?.title) { + fetchTranslations([res.post.id]); + } + + return res; + }, ...restOptions, staleTime: StaleTime.Default, enabled: !!id && tokenRefreshed, From 6343527a5a34779437090fef5396212a0c0ffe6d Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Thu, 23 Jan 2025 10:52:54 +0100 Subject: [PATCH 05/15] fix: mock TransformStream --- packages/extension/__tests__/setup.ts | 9 +++++++++ packages/shared/__tests__/setup.ts | 9 +++++++++ packages/webapp/__tests__/setup.ts | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/packages/extension/__tests__/setup.ts b/packages/extension/__tests__/setup.ts index 12cf7c8fdf..25dbd9656b 100644 --- a/packages/extension/__tests__/setup.ts +++ b/packages/extension/__tests__/setup.ts @@ -71,4 +71,13 @@ Object.defineProperty(global, 'open', { value: jest.fn(), }); +Object.defineProperty(global, 'TransformStream', { + writable: true, + value: jest.fn().mockImplementation(() => ({ + backpressure: jest.fn(), + readable: jest.fn(), + writable: jest.fn(), + })), +}); + structuredCloneJsonPolyfill(); diff --git a/packages/shared/__tests__/setup.ts b/packages/shared/__tests__/setup.ts index b79ec45a7a..40089ae541 100644 --- a/packages/shared/__tests__/setup.ts +++ b/packages/shared/__tests__/setup.ts @@ -53,6 +53,15 @@ Object.defineProperty(global, 'open', { value: jest.fn(), }); +Object.defineProperty(global, 'TransformStream', { + writable: true, + value: jest.fn().mockImplementation(() => ({ + backpressure: jest.fn(), + readable: jest.fn(), + writable: jest.fn(), + })), +}); + jest.mock('next/router', () => ({ useRouter: jest.fn().mockImplementation( () => diff --git a/packages/webapp/__tests__/setup.ts b/packages/webapp/__tests__/setup.ts index 6074bf1fab..40a855c517 100644 --- a/packages/webapp/__tests__/setup.ts +++ b/packages/webapp/__tests__/setup.ts @@ -38,6 +38,15 @@ Object.defineProperty(global, 'open', { value: jest.fn(), }); +Object.defineProperty(global, 'TransformStream', { + writable: true, + value: jest.fn().mockImplementation(() => ({ + backpressure: jest.fn(), + readable: jest.fn(), + writable: jest.fn(), + })), +}); + jest.mock('next/router', () => ({ useRouter: jest.fn().mockImplementation( () => From 087a1db898775d5b62c09d0a9b08abd00567663b Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Thu, 23 Jan 2025 17:37:35 +0100 Subject: [PATCH 06/15] fix: add abort signal to stream --- packages/shared/src/hooks/translation/useTranslation.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index ec12ff3523..452670baed 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -100,6 +100,7 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { const messages = await stream( `${apiUrl}/translate/post/title?${params}`, { + signal: abort.current?.signal, headers: { Authorization: `Bearer ${accessToken?.token}`, 'Content-Language': language as string, From 020414ffd40828ee258ef2aa4ee4f68747098625 Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Thu, 23 Jan 2025 18:05:43 +0100 Subject: [PATCH 07/15] fix: merge into `isStreamActive` --- packages/shared/src/hooks/translation/useTranslation.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index 452670baed..96d8f1cca3 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -49,6 +49,7 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { const queryClient = useQueryClient(); const { language } = user || {}; + const isStreamActive = !isLoggedIn || !language; const updateFeed = useCallback( (translatedPost: TranslateEvent) => { @@ -85,7 +86,7 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { const fetchTranslations = useCallback( async (postIds: string[]) => { - if (!isLoggedIn || !language) { + if (!isStreamActive) { return; } if (postIds.length === 0) { @@ -122,7 +123,7 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { }, [ accessToken?.token, - isLoggedIn, + isStreamActive, language, queryType, updateFeed, From d880ae109da100aa9e111aa78c306c47659f850a Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:36:12 +0100 Subject: [PATCH 08/15] fix: isStreamActive boolean logic --- packages/shared/src/hooks/translation/useTranslation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index 96d8f1cca3..766078a0cf 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -49,7 +49,7 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { const queryClient = useQueryClient(); const { language } = user || {}; - const isStreamActive = !isLoggedIn || !language; + const isStreamActive = isLoggedIn || !!language; const updateFeed = useCallback( (translatedPost: TranslateEvent) => { From 7fe2c72dee7e8e18a0a562ff5ea68d552980de54 Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Fri, 24 Jan 2025 14:57:08 +0100 Subject: [PATCH 09/15] fix: filter based on shared post title if no title exists --- packages/shared/src/hooks/useFeed.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/shared/src/hooks/useFeed.ts b/packages/shared/src/hooks/useFeed.ts index ab0e3fd458..497cbe1cdc 100644 --- a/packages/shared/src/hooks/useFeed.ts +++ b/packages/shared/src/hooks/useFeed.ts @@ -141,7 +141,11 @@ export default function useFeed( fetchTranslations( res.page.edges - .filter(({ node }) => !node?.translation?.title) + .filter(({ node }) => + node?.title + ? !node?.translation?.title + : !node?.sharedPost?.translation?.title, + ) .map(({ node }) => (node?.title ? node.id : node?.sharedPost.id)), ); From e40f6e68c513b6df795ce715e667a34bc2112d31 Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:22:54 +0100 Subject: [PATCH 10/15] refactor: replace `stream` with `events` --- .../src/hooks/translation/useTranslation.ts | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index 766078a0cf..2aa3791392 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -1,7 +1,7 @@ import { useCallback, useEffect, useRef } from 'react'; import type { InfiniteData, QueryKey } from '@tanstack/react-query'; import { useQueryClient } from '@tanstack/react-query'; -import { stream } from 'fetch-event-stream'; +import { events } from 'fetch-event-stream'; import { useAuthContext } from '../../contexts/AuthContext'; import { apiUrl } from '../../lib/config'; import type { FeedData, Post } from '../../graphql/posts'; @@ -98,19 +98,21 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { params.append('id', id); }); - const messages = await stream( - `${apiUrl}/translate/post/title?${params}`, - { - signal: abort.current?.signal, - headers: { - Authorization: `Bearer ${accessToken?.token}`, - 'Content-Language': language as string, - }, + const response = await fetch(`${apiUrl}/translate/post/title?${params}`, { + signal: abort.current?.signal, + headers: { + Accept: 'text/event-stream', + Authorization: `Bearer ${accessToken?.token}`, + 'Content-Language': language as string, }, - ); + }); + + if (!response.ok) { + return; + } // eslint-disable-next-line no-restricted-syntax - for await (const message of messages) { + for await (const message of events(response)) { if (message.event === ServerEvents.Message) { const post = JSON.parse(message.data) as TranslateEvent; if (queryType === 'feed') { From 3cc900b1eb8d751c485b9dd0a5d8f1486684086b Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:26:03 +0100 Subject: [PATCH 11/15] fix: require language in isStreamActive --- packages/shared/src/hooks/translation/useTranslation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index 2aa3791392..b055d08c5a 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -49,7 +49,7 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { const queryClient = useQueryClient(); const { language } = user || {}; - const isStreamActive = isLoggedIn || !!language; + const isStreamActive = isLoggedIn && !!language; const updateFeed = useCallback( (translatedPost: TranslateEvent) => { From 928af03951d09b70655e1abbc43e22f7c7fd78ea Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:37:26 +0100 Subject: [PATCH 12/15] fix: force memo to re-run when feed query is updated --- packages/shared/src/hooks/useFeed.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/shared/src/hooks/useFeed.ts b/packages/shared/src/hooks/useFeed.ts index 497cbe1cdc..0ea8fd4d54 100644 --- a/packages/shared/src/hooks/useFeed.ts +++ b/packages/shared/src/hooks/useFeed.ts @@ -30,6 +30,7 @@ import { useTranslation } from './translation/useTranslation'; interface FeedItemBase { type: T; + dataUpdatedAt: number; } interface AdItem extends FeedItemBase { @@ -304,6 +305,7 @@ export default function useFeed( post: node, page: pageIndex, index, + dataUpdatedAt: feedQuery.dataUpdatedAt, }); }); @@ -319,6 +321,7 @@ export default function useFeed( }, [ feedQuery.data, feedQuery.isFetching, + feedQuery.dataUpdatedAt, settings.marketingCta, settings.showAcquisitionForm, placeholdersPerPage, From a81d57a403b75bc6b83003023a72cdec349f94b8 Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Mon, 27 Jan 2025 06:36:58 +0100 Subject: [PATCH 13/15] refactor: move filter logic into hook --- .../src/hooks/translation/useTranslation.ts | 22 +++++++++++++++++-- packages/shared/src/hooks/useFeed.ts | 10 +-------- packages/shared/src/hooks/usePostById.ts | 2 +- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index b055d08c5a..5946e8b380 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -22,7 +22,7 @@ type UseTranslation = (props: { queryKey: QueryKey; queryType: 'post' | 'feed'; }) => { - fetchTranslations: (id: string[]) => void; + fetchTranslations: (id: Post[]) => void; }; type TranslateEvent = { @@ -85,10 +85,28 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { ); const fetchTranslations = useCallback( - async (postIds: string[]) => { + async (posts: Post[]) => { if (!isStreamActive) { return; } + if (posts.length === 0) { + return; + } + + const postIds = posts + .filter((node) => + node?.title + ? !node?.translation?.title + : !node?.sharedPost?.translation?.title, + ) + .filter((node) => + node?.title + ? !node.clickbaitTitleDetected + : !node.sharedPost?.clickbaitTitleDetected, + ) + .filter(Boolean) + .map((node) => (node?.title ? node.id : node?.sharedPost.id)); + if (postIds.length === 0) { return; } diff --git a/packages/shared/src/hooks/useFeed.ts b/packages/shared/src/hooks/useFeed.ts index 0ea8fd4d54..8204b48179 100644 --- a/packages/shared/src/hooks/useFeed.ts +++ b/packages/shared/src/hooks/useFeed.ts @@ -140,15 +140,7 @@ export default function useFeed( } } - fetchTranslations( - res.page.edges - .filter(({ node }) => - node?.title - ? !node?.translation?.title - : !node?.sharedPost?.translation?.title, - ) - .map(({ node }) => (node?.title ? node.id : node?.sharedPost.id)), - ); + fetchTranslations(res.page.edges.map(({ node }) => node)); return res; }, diff --git a/packages/shared/src/hooks/usePostById.ts b/packages/shared/src/hooks/usePostById.ts index 4190a68ab7..fe65def3a7 100644 --- a/packages/shared/src/hooks/usePostById.ts +++ b/packages/shared/src/hooks/usePostById.ts @@ -108,7 +108,7 @@ const usePostById = ({ id, options = {} }: UsePostByIdProps): UsePostById => { queryFn: async () => { const res = await gqlClient.request(POST_BY_ID_QUERY, { id }); if (!res.post?.translation?.title) { - fetchTranslations([res.post.id]); + fetchTranslations([res.post]); } return res; From 82fc409c741011b2d7ac58a53037db8061f6ed7d Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Mon, 27 Jan 2025 13:19:16 +0100 Subject: [PATCH 14/15] fix: fetch translation for original title when clickbait shield is disabled --- packages/shared/src/hooks/translation/useTranslation.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index 5946e8b380..cc0baeb3ff 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -10,6 +10,7 @@ import { findIndexOfPostInData, updatePostCache, } from '../../lib/query'; +import { useSettingsContext } from '../../contexts/SettingsContext'; export enum ServerEvents { Connect = 'connect', @@ -46,6 +47,7 @@ const updateTranslation = (post: Post, translation: TranslateEvent): Post => { export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { const abort = useRef(); const { user, accessToken, isLoggedIn } = useAuthContext(); + const { flags } = useSettingsContext(); const queryClient = useQueryClient(); const { language } = user || {}; @@ -100,7 +102,7 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { : !node?.sharedPost?.translation?.title, ) .filter((node) => - node?.title + flags.clickbaitShieldEnabled && node?.title ? !node.clickbaitTitleDetected : !node.sharedPost?.clickbaitTitleDetected, ) From 84ccf8b919feece9f82681a60a3bd664a4f24832 Mon Sep 17 00:00:00 2001 From: Ole-Martin Bratteng <1681525+omBratteng@users.noreply.github.com> Date: Mon, 27 Jan 2025 13:30:43 +0100 Subject: [PATCH 15/15] fix: lint --- packages/shared/src/hooks/translation/useTranslation.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/shared/src/hooks/translation/useTranslation.ts b/packages/shared/src/hooks/translation/useTranslation.ts index cc0baeb3ff..231cfcb635 100644 --- a/packages/shared/src/hooks/translation/useTranslation.ts +++ b/packages/shared/src/hooks/translation/useTranslation.ts @@ -145,6 +145,7 @@ export const useTranslation: UseTranslation = ({ queryKey, queryType }) => { }, [ accessToken?.token, + flags.clickbaitShieldEnabled, isStreamActive, language, queryType,