From 32689dc647f97f545da7f439612218eee38d3b99 Mon Sep 17 00:00:00 2001 From: bill Date: Thu, 28 Nov 2024 14:04:20 +0800 Subject: [PATCH] Reapply "Feat: Scrolling knowledge base list #3695" (#3708) This reverts commit 9f57534843c435aaaaa07eb2f13c0660fa7940f8. --- api/apps/kb_app.py | 7 ++- api/db/services/knowledgebase_service.py | 27 ++++++--- web/.umirc.ts | 2 +- web/package-lock.json | 20 +++++++ web/package.json | 1 + web/src/hooks/knowledge-hooks.ts | 55 ++++++++++++++++- web/src/locales/en.ts | 1 + web/src/locales/zh-traditional.ts | 1 + web/src/locales/zh.ts | 1 + web/src/pages/knowledge/index.less | 1 + web/src/pages/knowledge/index.tsx | 75 +++++++++++++++++------- 11 files changed, 155 insertions(+), 36 deletions(-) diff --git a/api/apps/kb_app.py b/api/apps/kb_app.py index ebac350f10b..fa867493245 100644 --- a/api/apps/kb_app.py +++ b/api/apps/kb_app.py @@ -125,15 +125,16 @@ def detail(): @manager.route('/list', methods=['GET']) @login_required def list_kbs(): + keywords = request.args.get("keywords", "") page_number = int(request.args.get("page", 1)) items_per_page = int(request.args.get("page_size", 150)) orderby = request.args.get("orderby", "create_time") desc = request.args.get("desc", True) try: tenants = TenantService.get_joined_tenants_by_user_id(current_user.id) - kbs = KnowledgebaseService.get_by_tenant_ids( - [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc) - return get_json_result(data=kbs) + kbs, total = KnowledgebaseService.get_by_tenant_ids( + [m["tenant_id"] for m in tenants], current_user.id, page_number, items_per_page, orderby, desc, keywords) + return get_json_result(data={"kbs": kbs, "total": total}) except Exception as e: return server_error_response(e) diff --git a/api/db/services/knowledgebase_service.py b/api/db/services/knowledgebase_service.py index 1b4c82cf178..e05a14a166d 100644 --- a/api/db/services/knowledgebase_service.py +++ b/api/db/services/knowledgebase_service.py @@ -34,7 +34,7 @@ def list_documents_by_ids(cls,kb_ids): @classmethod @DB.connection_context() def get_by_tenant_ids(cls, joined_tenant_ids, user_id, - page_number, items_per_page, orderby, desc): + page_number, items_per_page, orderby, desc, keywords): fields = [ cls.model.id, cls.model.avatar, @@ -51,20 +51,31 @@ def get_by_tenant_ids(cls, joined_tenant_ids, user_id, User.avatar.alias('tenant_avatar'), cls.model.update_time ] - kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where( - ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission == - TenantPermission.TEAM.value)) | ( - cls.model.tenant_id == user_id)) - & (cls.model.status == StatusEnum.VALID.value) - ) + if keywords: + kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where( + ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission == + TenantPermission.TEAM.value)) | ( + cls.model.tenant_id == user_id)) + & (cls.model.status == StatusEnum.VALID.value), + (fn.LOWER(cls.model.name).contains(keywords.lower())) + ) + else: + kbs = cls.model.select(*fields).join(User, on=(cls.model.tenant_id == User.id)).where( + ((cls.model.tenant_id.in_(joined_tenant_ids) & (cls.model.permission == + TenantPermission.TEAM.value)) | ( + cls.model.tenant_id == user_id)) + & (cls.model.status == StatusEnum.VALID.value) + ) if desc: kbs = kbs.order_by(cls.model.getter_by(orderby).desc()) else: kbs = kbs.order_by(cls.model.getter_by(orderby).asc()) + count = kbs.count() + kbs = kbs.paginate(page_number, items_per_page) - return list(kbs.dicts()) + return list(kbs.dicts()), count @classmethod @DB.connection_context() diff --git a/web/.umirc.ts b/web/.umirc.ts index 8de9ff1e152..b96d84c1bb9 100644 --- a/web/.umirc.ts +++ b/web/.umirc.ts @@ -34,7 +34,7 @@ export default defineConfig({ proxy: [ { context: ['/api', '/v1'], - target: 'http://127.0.0.1:9456/', + target: 'http://127.0.0.1:9380/', changeOrigin: true, ws: true, logger: console, diff --git a/web/package-lock.json b/web/package-lock.json index 779be63867c..65421945fba 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -57,6 +57,7 @@ "react-force-graph": "^1.44.4", "react-hook-form": "^7.53.1", "react-i18next": "^14.0.0", + "react-infinite-scroll-component": "^6.1.0", "react-markdown": "^9.0.1", "react-pdf-highlighter": "^6.1.0", "react-string-replace": "^1.1.1", @@ -24705,6 +24706,25 @@ } } }, + "node_modules/react-infinite-scroll-component": { + "version": "6.1.0", + "resolved": "https://registry.npmmirror.com/react-infinite-scroll-component/-/react-infinite-scroll-component-6.1.0.tgz", + "integrity": "sha512-SQu5nCqy8DxQWpnUVLx7V7b7LcA37aM7tvoWjTLZp1dk6EJibM5/4EJKzOnl07/BsM1Y40sKLuqjCwwH/xV0TQ==", + "dependencies": { + "throttle-debounce": "^2.1.0" + }, + "peerDependencies": { + "react": ">=16.0.0" + } + }, + "node_modules/react-infinite-scroll-component/node_modules/throttle-debounce": { + "version": "2.3.0", + "resolved": "https://registry.npmmirror.com/throttle-debounce/-/throttle-debounce-2.3.0.tgz", + "integrity": "sha512-H7oLPV0P7+jgvrk+6mwwwBDmxTaxnu9HMXmloNLXwnNO0ZxZ31Orah2n8lU1eMPvsaowP2CX+USCgyovXfdOFQ==", + "engines": { + "node": ">=8" + } + }, "node_modules/react-is": { "version": "18.2.0", "resolved": "https://registry.npmmirror.com/react-is/-/react-is-18.2.0.tgz", diff --git a/web/package.json b/web/package.json index 061b17352fa..3a226d97f8a 100644 --- a/web/package.json +++ b/web/package.json @@ -68,6 +68,7 @@ "react-force-graph": "^1.44.4", "react-hook-form": "^7.53.1", "react-i18next": "^14.0.0", + "react-infinite-scroll-component": "^6.1.0", "react-markdown": "^9.0.1", "react-pdf-highlighter": "^6.1.0", "react-string-replace": "^1.1.1", diff --git a/web/src/hooks/knowledge-hooks.ts b/web/src/hooks/knowledge-hooks.ts index 0dc245c06d1..8b460c123b8 100644 --- a/web/src/hooks/knowledge-hooks.ts +++ b/web/src/hooks/knowledge-hooks.ts @@ -3,14 +3,17 @@ import { IKnowledge, ITestingResult } from '@/interfaces/database/knowledge'; import i18n from '@/locales/config'; import kbService from '@/services/knowledge-service'; import { + useInfiniteQuery, useIsMutating, useMutation, useMutationState, useQuery, useQueryClient, } from '@tanstack/react-query'; +import { useDebounce } from 'ahooks'; import { message } from 'antd'; import { useSearchParams } from 'umi'; +import { useHandleSearchChange } from './logic-hooks'; import { useSetPaginationParams } from './route-hook'; export const useKnowledgeBaseId = (): string => { @@ -50,7 +53,7 @@ export const useNextFetchKnowledgeList = ( gcTime: 0, // https://tanstack.com/query/latest/docs/framework/react/guides/caching?from=reactQueryV3 queryFn: async () => { const { data } = await kbService.getList(); - const list = data?.data ?? []; + const list = data?.data?.kbs ?? []; return shouldFilterListWithoutDocument ? list.filter((x: IKnowledge) => x.chunk_num > 0) : list; @@ -60,6 +63,52 @@ export const useNextFetchKnowledgeList = ( return { list: data, loading }; }; +export const useInfiniteFetchKnowledgeList = () => { + const { searchString, handleInputChange } = useHandleSearchChange(); + const debouncedSearchString = useDebounce(searchString, { wait: 500 }); + + const PageSize = 10; + const { + data, + error, + fetchNextPage, + hasNextPage, + isFetching, + isFetchingNextPage, + status, + } = useInfiniteQuery({ + queryKey: ['infiniteFetchKnowledgeList', debouncedSearchString], + queryFn: async ({ pageParam }) => { + const { data } = await kbService.getList({ + page: pageParam, + page_size: PageSize, + keywords: debouncedSearchString, + }); + const list = data?.data ?? []; + return list; + }, + initialPageParam: 1, + getNextPageParam: (lastPage, pages, lastPageParam) => { + if (lastPageParam * PageSize <= lastPage.total) { + return lastPageParam + 1; + } + return undefined; + }, + }); + return { + data, + loading: isFetching, + error, + fetchNextPage, + hasNextPage, + isFetching, + isFetchingNextPage, + status, + handleInputChange, + searchString, + }; +}; + export const useCreateKnowledge = () => { const queryClient = useQueryClient(); const { @@ -95,7 +144,9 @@ export const useDeleteKnowledge = () => { const { data } = await kbService.rmKb({ kb_id: id }); if (data.code === 0) { message.success(i18n.t(`message.deleted`)); - queryClient.invalidateQueries({ queryKey: ['fetchKnowledgeList'] }); + queryClient.invalidateQueries({ + queryKey: ['infiniteFetchKnowledgeList'], + }); } return data?.data ?? []; }, diff --git a/web/src/locales/en.ts b/web/src/locales/en.ts index 57253d33a31..59b8d0871f8 100644 --- a/web/src/locales/en.ts +++ b/web/src/locales/en.ts @@ -75,6 +75,7 @@ export default { namePlaceholder: 'Please input name!', doc: 'Docs', searchKnowledgePlaceholder: 'Search', + noMoreData: 'It is all, nothing more', }, knowledgeDetails: { dataset: 'Dataset', diff --git a/web/src/locales/zh-traditional.ts b/web/src/locales/zh-traditional.ts index 9f92d9ffaa5..bd9602e97ee 100644 --- a/web/src/locales/zh-traditional.ts +++ b/web/src/locales/zh-traditional.ts @@ -75,6 +75,7 @@ export default { namePlaceholder: '請輸入名稱', doc: '文件', searchKnowledgePlaceholder: '搜索', + noMoreData: 'It is all, nothing more', }, knowledgeDetails: { dataset: '數據集', diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.ts index 59c0639ac75..6ffdf246464 100644 --- a/web/src/locales/zh.ts +++ b/web/src/locales/zh.ts @@ -75,6 +75,7 @@ export default { namePlaceholder: '请输入名称', doc: '文档', searchKnowledgePlaceholder: '搜索', + noMoreData: '沒有更多的數據了', }, knowledgeDetails: { dataset: '数据集', diff --git a/web/src/pages/knowledge/index.less b/web/src/pages/knowledge/index.less index 2479453fa1f..25bcb00deb1 100644 --- a/web/src/pages/knowledge/index.less +++ b/web/src/pages/knowledge/index.less @@ -2,6 +2,7 @@ .knowledge { padding: 48px 0; + overflow: auto; } .topWrapper { diff --git a/web/src/pages/knowledge/index.tsx b/web/src/pages/knowledge/index.tsx index 64c1d6c73ba..d41d6416d2e 100644 --- a/web/src/pages/knowledge/index.tsx +++ b/web/src/pages/knowledge/index.tsx @@ -1,18 +1,26 @@ -import { useNextFetchKnowledgeList } from '@/hooks/knowledge-hooks'; +import { useInfiniteFetchKnowledgeList } from '@/hooks/knowledge-hooks'; import { useFetchUserInfo } from '@/hooks/user-setting-hooks'; import { PlusOutlined, SearchOutlined } from '@ant-design/icons'; -import { Button, Empty, Flex, Input, Space, Spin } from 'antd'; +import { + Button, + Divider, + Empty, + Flex, + Input, + Skeleton, + Space, + Spin, +} from 'antd'; +import { useTranslation } from 'react-i18next'; +import InfiniteScroll from 'react-infinite-scroll-component'; +import { useSaveKnowledge } from './hooks'; import KnowledgeCard from './knowledge-card'; import KnowledgeCreatingModal from './knowledge-creating-modal'; -import { useTranslation } from 'react-i18next'; -import { useSaveKnowledge, useSearchKnowledge } from './hooks'; +import { useMemo } from 'react'; import styles from './index.less'; const KnowledgeList = () => { - const { searchString, handleInputChange } = useSearchKnowledge(); - const { loading, list: data } = useNextFetchKnowledgeList(); - const list = data.filter((x) => x.name.includes(searchString)); const { data: userInfo } = useFetchUserInfo(); const { t } = useTranslation('translation', { keyPrefix: 'knowledgeList' }); const { @@ -22,9 +30,23 @@ const KnowledgeList = () => { onCreateOk, loading: creatingLoading, } = useSaveKnowledge(); + const { + fetchNextPage, + data, + hasNextPage, + searchString, + handleInputChange, + loading, + } = useInfiniteFetchKnowledgeList(); + console.log('🚀 ~ KnowledgeList ~ data:', data); + const nextList = data?.pages?.flatMap((x) => x.kbs) ?? []; + + const total = useMemo(() => { + return data?.pages.at(-1).total ?? 0; + }, [data?.pages]); return ( - +
@@ -53,21 +75,30 @@ const KnowledgeList = () => {
- } + endMessage={total && {t('noMoreData')} 🤐} + scrollableTarget="scrollableDiv" > - {list.length > 0 ? ( - list.map((item: any) => { - return ( - - ); - }) - ) : ( - - )} - + + {nextList?.length > 0 ? ( + nextList.map((item: any) => { + return ( + + ); + }) + ) : ( + + )} + +