diff --git a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx index a6fb116fa8a50f..b416659a6a1cfa 100644 --- a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx +++ b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx @@ -7,36 +7,85 @@ import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' import { Cog8ToothIcon, + // CommandLineIcon, + Squares2X2Icon, + // eslint-disable-next-line sort-imports + PuzzlePieceIcon, DocumentTextIcon, PaperClipIcon, + QuestionMarkCircleIcon, } from '@heroicons/react/24/outline' import { Cog8ToothIcon as Cog8ToothSolidIcon, // CommandLineIcon as CommandLineSolidIcon, DocumentTextIcon as DocumentTextSolidIcon, } from '@heroicons/react/24/solid' -import { RiApps2AddLine, RiInformation2Line } from '@remixicon/react' +import Link from 'next/link' import s from './style.module.css' import classNames from '@/utils/classnames' import { fetchDatasetDetail, fetchDatasetRelatedApps } from '@/service/datasets' -import type { RelatedAppResponse } from '@/models/datasets' +import type { RelatedApp, RelatedAppResponse } from '@/models/datasets' import AppSideBar from '@/app/components/app-sidebar' +import Divider from '@/app/components/base/divider' +import AppIcon from '@/app/components/base/app-icon' import Loading from '@/app/components/base/loading' +import FloatPopoverContainer from '@/app/components/base/float-popover-container' import DatasetDetailContext from '@/context/dataset-detail' import { DataSourceType } from '@/models/datasets' import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' import { LanguagesSupported } from '@/i18n/language' import { useStore } from '@/app/components/app/store' +import { AiText, ChatBot, CuteRobot } from '@/app/components/base/icons/src/vender/solid/communication' +import { Route } from '@/app/components/base/icons/src/vender/solid/mapsAndTravel' import { getLocaleOnClient } from '@/i18n' import { useAppContext } from '@/context/app-context' -import Tooltip from '@/app/components/base/tooltip' -import LinkedAppsPanel from '@/app/components/base/linked-apps-panel' export type IAppDetailLayoutProps = { children: React.ReactNode params: { datasetId: string } } +type ILikedItemProps = { + type?: 'plugin' | 'app' + appStatus?: boolean + detail: RelatedApp + isMobile: boolean +} + +const LikedItem = ({ + type = 'app', + detail, + isMobile, +}: ILikedItemProps) => { + return ( + +
+ + {type === 'app' && ( + + {detail.mode === 'advanced-chat' && ( + + )} + {detail.mode === 'agent-chat' && ( + + )} + {detail.mode === 'chat' && ( + + )} + {detail.mode === 'completion' && ( + + )} + {detail.mode === 'workflow' && ( + + )} + + )} +
+ {!isMobile &&
{detail?.name || '--'}
} + + ) +} + const TargetIcon = ({ className }: SVGProps) => { return @@ -68,80 +117,65 @@ const BookOpenIcon = ({ className }: SVGProps) => { type IExtraInfoProps = { isMobile: boolean relatedApps?: RelatedAppResponse - expand: boolean } -const ExtraInfo = ({ isMobile, relatedApps, expand }: IExtraInfoProps) => { +const ExtraInfo = ({ isMobile, relatedApps }: IExtraInfoProps) => { const locale = getLocaleOnClient() const [isShowTips, { toggle: toggleTips, set: setShowTips }] = useBoolean(!isMobile) const { t } = useTranslation() - const hasRelatedApps = relatedApps?.data && relatedApps?.data?.length > 0 - const relatedAppsTotal = relatedApps?.data?.length || 0 - useEffect(() => { setShowTips(!isMobile) }, [isMobile, setShowTips]) - return
- {hasRelatedApps && ( + return
+ + {(relatedApps?.data && relatedApps?.data?.length > 0) && ( <> - {!isMobile && ( - - } - > -
- {relatedAppsTotal || '--'} {t('common.datasetMenus.relatedApp')} - -
- - )} - + {!isMobile &&
{relatedApps?.total || '--'} {t('common.datasetMenus.relatedApp')}
} {isMobile &&
- {relatedAppsTotal || '--'} + {relatedApps?.total || '--'}
} + {relatedApps?.data?.map((item, index) => ())} )} - {!hasRelatedApps && !expand && ( - -
- -
-
{t('common.datasetMenus.emptyTip')}
- - - {t('common.datasetMenus.viewDoc')} - + {!relatedApps?.data?.length && ( + +
} > -
- {t('common.datasetMenus.noRelatedApp')} - +
+
+
+ +
+
+ +
+
+
{t('common.datasetMenus.emptyTip')}
+ + + {t('common.datasetMenus.viewDoc')} +
- + )}
} @@ -201,7 +235,7 @@ const DatasetDetailLayout: FC = (props) => { }, [isMobile, setAppSiderbarExpand]) if (!datasetRes && !error) - return + return return (
@@ -212,7 +246,7 @@ const DatasetDetailLayout: FC = (props) => { desc={datasetRes?.description || '--'} isExternal={datasetRes?.provider === 'external'} navigation={navigation} - extraInfo={!isCurrentWorkspaceDatasetOperator ? mode => : undefined} + extraInfo={!isCurrentWorkspaceDatasetOperator ? mode => : undefined} iconType={datasetRes?.data_source_type === DataSourceType.NOTION ? 'notion' : 'dataset'} />} = (props) => { dataset: datasetRes, mutateDatasetRes: () => mutateDatasetRes(), }}> -
{children}
+
{children}
) diff --git a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/settings/page.tsx b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/settings/page.tsx index 3a65f1d30ff7ef..df314ddafe9d15 100644 --- a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/settings/page.tsx +++ b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/settings/page.tsx @@ -7,10 +7,10 @@ const Settings = async () => { const { t } = await translate(locale, 'dataset-settings') return ( -
+
-
{t('title')}
-
{t('desc')}
+
{t('title')}
+
{t('desc')}
diff --git a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/style.module.css b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/style.module.css index 516b124809b9be..0ee64b4fcd0f82 100644 --- a/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/style.module.css +++ b/web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/style.module.css @@ -1,3 +1,12 @@ +.itemWrapper { + @apply flex items-center w-full h-10 rounded-lg hover:bg-gray-50 cursor-pointer; +} +.appInfo { + @apply truncate text-gray-700 text-sm font-normal; +} +.iconWrapper { + @apply relative w-6 h-6 rounded-lg; +} .statusPoint { @apply flex justify-center items-center absolute -right-0.5 -bottom-0.5 w-2.5 h-2.5 bg-white rounded; } diff --git a/web/app/(commonLayout)/datasets/Container.tsx b/web/app/(commonLayout)/datasets/Container.tsx index a0edb1cd61ba3c..a30521d9988a5e 100644 --- a/web/app/(commonLayout)/datasets/Container.tsx +++ b/web/app/(commonLayout)/datasets/Container.tsx @@ -17,6 +17,7 @@ import TagManagementModal from '@/app/components/base/tag-management' import TagFilter from '@/app/components/base/tag-management/filter' import Button from '@/app/components/base/button' import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development' +import SearchInput from '@/app/components/base/search-input' // Services import { fetchDatasetApiBaseUrl } from '@/service/datasets' @@ -28,7 +29,6 @@ import { useAppContext } from '@/context/app-context' import { useExternalApiPanel } from '@/context/external-api-panel-context' // eslint-disable-next-line import/order import { useQuery } from '@tanstack/react-query' -import Input from '@/app/components/base/input' const Container = () => { const { t } = useTranslation() @@ -81,24 +81,17 @@ const Container = () => { }, [currentWorkspace, router]) return ( -
-
+
+
setActiveTab(newActiveTab)} options={options} /> {activeTab === 'dataset' && ( -
+
- handleKeywordsChange(e.target.value)} - onClear={() => handleKeywordsChange('')} - /> +
- -
-
-} diff --git a/web/app/components/base/linked-apps-panel/index.tsx b/web/app/components/base/linked-apps-panel/index.tsx deleted file mode 100644 index 4320cb0fc6389f..00000000000000 --- a/web/app/components/base/linked-apps-panel/index.tsx +++ /dev/null @@ -1,62 +0,0 @@ -'use client' -import type { FC } from 'react' -import React from 'react' -import Link from 'next/link' -import { useTranslation } from 'react-i18next' -import { RiArrowRightUpLine } from '@remixicon/react' -import cn from '@/utils/classnames' -import AppIcon from '@/app/components/base/app-icon' -import type { RelatedApp } from '@/models/datasets' - -type ILikedItemProps = { - appStatus?: boolean - detail: RelatedApp - isMobile: boolean -} - -const appTypeMap = { - 'chat': 'Chatbot', - 'completion': 'Completion', - 'agent-chat': 'Agent', - 'advanced-chat': 'Chatflow', - 'workflow': 'Workflow', -} - -const LikedItem = ({ - detail, - isMobile, -}: ILikedItemProps) => { - return ( - -
-
- -
- {!isMobile &&
{detail?.name || '--'}
} -
-
{appTypeMap[detail.mode]}
- - - ) -} - -type Props = { - relatedApps: RelatedApp[] - isMobile: boolean -} - -const LinkedAppsPanel: FC = ({ - relatedApps, - isMobile, -}) => { - const { t } = useTranslation() - return ( -
-
{relatedApps.length || '--'} {t('common.datasetMenus.relatedApp')}
- {relatedApps.map((item, index) => ( - - ))} -
- ) -} -export default React.memo(LinkedAppsPanel) diff --git a/web/app/components/base/pagination/index.tsx b/web/app/components/base/pagination/index.tsx index c0cc9f86ec2852..b64c712425178a 100644 --- a/web/app/components/base/pagination/index.tsx +++ b/web/app/components/base/pagination/index.tsx @@ -8,7 +8,7 @@ import Button from '@/app/components/base/button' import Input from '@/app/components/base/input' import cn from '@/utils/classnames' -export type Props = { +type Props = { className?: string current: number onChange: (cur: number) => void diff --git a/web/app/components/base/param-item/index.tsx b/web/app/components/base/param-item/index.tsx index 68c980ad095b76..49acc8148455e5 100644 --- a/web/app/components/base/param-item/index.tsx +++ b/web/app/components/base/param-item/index.tsx @@ -1,6 +1,5 @@ 'use client' import type { FC } from 'react' -import { InputNumber } from '../input-number' import Tooltip from '@/app/components/base/tooltip' import Slider from '@/app/components/base/slider' import Switch from '@/app/components/base/switch' @@ -24,44 +23,39 @@ type Props = { const ParamItem: FC = ({ className, id, name, noTooltip, tip, step = 0.1, min = 0, max, value, enable, onChange, hasSwitch, onSwitchChange }) => { return (
-
-
+
+
{hasSwitch && ( { onSwitchChange?.(id, val) }} /> )} - {name} + {name} {!noTooltip && ( {tip}
} /> )} +
+
-
-
- { - onChange(id, value) - }} - className='w-[72px]' - /> +
+
+ { + const value = parseFloat(e.target.value) + if (value < min || value > max) + return + + onChange(id, value) + }} />
-
+
= ({ onChosen = () => { }, chosenConfig, chosenConfigWrapClassName, - className, }) => { return (
-
-
+
+
{icon}
-
{title}
-
{description}
+
{title}
+
{description}
{!noRadio && ( -
+
= ({ )}
{((isChosen && chosenConfig) || noRadio) && ( -
-
-
- {chosenConfig} -
+
+ {chosenConfig}
)}
diff --git a/web/app/components/base/retry-button/index.tsx b/web/app/components/base/retry-button/index.tsx new file mode 100644 index 00000000000000..689827af7b3afa --- /dev/null +++ b/web/app/components/base/retry-button/index.tsx @@ -0,0 +1,85 @@ +'use client' +import type { FC } from 'react' +import React, { useEffect, useReducer } from 'react' +import { useTranslation } from 'react-i18next' +import useSWR from 'swr' +import s from './style.module.css' +import classNames from '@/utils/classnames' +import Divider from '@/app/components/base/divider' +import { getErrorDocs, retryErrorDocs } from '@/service/datasets' +import type { IndexingStatusResponse } from '@/models/datasets' + +const WarningIcon = () => + + + + +type Props = { + datasetId: string +} +type IIndexState = { + value: string +} +type ActionType = 'retry' | 'success' | 'error' + +type IAction = { + type: ActionType +} +const indexStateReducer = (state: IIndexState, action: IAction) => { + const actionMap = { + retry: 'retry', + success: 'success', + error: 'error', + } + + return { + ...state, + value: actionMap[action.type] || state.value, + } +} + +const RetryButton: FC = ({ datasetId }) => { + const { t } = useTranslation() + const [indexState, dispatch] = useReducer(indexStateReducer, { value: 'success' }) + const { data: errorDocs } = useSWR({ datasetId }, getErrorDocs) + + const onRetryErrorDocs = async () => { + dispatch({ type: 'retry' }) + const document_ids = errorDocs?.data.map((doc: IndexingStatusResponse) => doc.id) || [] + const res = await retryErrorDocs({ datasetId, document_ids }) + if (res.result === 'success') + dispatch({ type: 'success' }) + else + dispatch({ type: 'error' }) + } + + useEffect(() => { + if (errorDocs?.total === 0) + dispatch({ type: 'success' }) + else + dispatch({ type: 'error' }) + }, [errorDocs?.total]) + + if (indexState.value === 'success') + return null + + return ( +
+ + + {errorDocs?.total} {t('dataset.docsFailedNotice')} + + + + {t('dataset.retry')} + +
+ ) +} +export default RetryButton diff --git a/web/app/components/base/retry-button/style.module.css b/web/app/components/base/retry-button/style.module.css new file mode 100644 index 00000000000000..99a0947576d9c2 --- /dev/null +++ b/web/app/components/base/retry-button/style.module.css @@ -0,0 +1,4 @@ +.retryBtn { + @apply inline-flex justify-center items-center content-center h-9 leading-5 rounded-lg px-4 py-2 text-base; + @apply border-solid border border-gray-200 text-gray-500 hover:bg-white hover:shadow-sm hover:border-gray-300; +} diff --git a/web/app/components/base/simple-pie-chart/index.tsx b/web/app/components/base/simple-pie-chart/index.tsx index 4b987ab42dec7d..7de539cbb1ac45 100644 --- a/web/app/components/base/simple-pie-chart/index.tsx +++ b/web/app/components/base/simple-pie-chart/index.tsx @@ -10,11 +10,10 @@ export type SimplePieChartProps = { fill?: string stroke?: string size?: number - animationDuration?: number className?: string } -const SimplePieChart = ({ percentage = 80, fill = '#fdb022', stroke = '#f79009', size = 12, animationDuration, className }: SimplePieChartProps) => { +const SimplePieChart = ({ percentage = 80, fill = '#fdb022', stroke = '#f79009', size = 12, className }: SimplePieChartProps) => { const option: EChartsOption = useMemo(() => ({ series: [ { @@ -35,7 +34,7 @@ const SimplePieChart = ({ percentage = 80, fill = '#fdb022', stroke = '#f79009', { type: 'pie', radius: '83%', - animationDuration: animationDuration ?? 600, + animationDuration: 600, data: [ { value: percentage, itemStyle: { color: fill } }, { value: 100 - percentage, itemStyle: { color: '#fff' } }, @@ -49,7 +48,7 @@ const SimplePieChart = ({ percentage = 80, fill = '#fdb022', stroke = '#f79009', cursor: 'default', }, ], - }), [stroke, fill, percentage, animationDuration]) + }), [stroke, fill, percentage]) return ( -export const SkeletonContainer: FC = (props) => { +export const SkeletonContanier: FC = (props) => { const { className, children, ...rest } = props return (
@@ -30,14 +30,11 @@ export const SkeletonRectangle: FC = (props) => { ) } -export const SkeletonPoint: FC = (props) => { - const { className, ...rest } = props - return ( -
·
- ) -} +export const SkeletonPoint: FC = () => +
·
+ /** Usage - * + * * * * diff --git a/web/app/components/base/switch/index.tsx b/web/app/components/base/switch/index.tsx index 48e5c0cd8c169c..2b7742f185845e 100644 --- a/web/app/components/base/switch/index.tsx +++ b/web/app/components/base/switch/index.tsx @@ -67,7 +67,4 @@ const Switch = React.forwardRef( ) }) - -Switch.displayName = 'Switch' - export default React.memo(Switch) diff --git a/web/app/components/base/tag-input/index.tsx b/web/app/components/base/tag-input/index.tsx index ec6c1cee342665..b26d0c6438c067 100644 --- a/web/app/components/base/tag-input/index.tsx +++ b/web/app/components/base/tag-input/index.tsx @@ -3,8 +3,8 @@ import type { ChangeEvent, FC, KeyboardEvent } from 'react' import { } from 'use-context-selector' import { useTranslation } from 'react-i18next' import AutosizeInput from 'react-18-input-autosize' -import { RiAddLine, RiCloseLine } from '@remixicon/react' import cn from '@/utils/classnames' +import { X } from '@/app/components/base/icons/src/vender/line/general' import { useToastContext } from '@/app/components/base/toast' type TagInputProps = { @@ -75,14 +75,14 @@ const TagInput: FC = ({ (items || []).map((item, index) => (
+ className={cn('flex items-center mr-1 mt-1 px-2 py-1 text-sm text-gray-700 border border-gray-200', isSpecialMode ? 'bg-white rounded-md' : 'rounded-lg')}> {item} { !disableRemove && ( -
handleRemove(index)}> - -
+ handleRemove(index)} + /> ) }
@@ -90,27 +90,24 @@ const TagInput: FC = ({ } { !disableAdd && ( -
- {!isSpecialMode && !focused && } - setFocused(true)} - onBlur={handleBlur} - value={value} - onChange={(e: ChangeEvent) => { - setValue(e.target.value) - }} - onKeyDown={handleKeyDown} - placeholder={t(placeholder || (isSpecialMode ? 'common.model.params.stop_sequencesPlaceholder' : 'datasetDocuments.segment.addKeyWord'))} - /> -
+ setFocused(true)} + onBlur={handleBlur} + value={value} + onChange={(e: ChangeEvent) => { + setValue(e.target.value) + }} + onKeyDown={handleKeyDown} + placeholder={t(placeholder || (isSpecialMode ? 'common.model.params.stop_sequencesPlaceholder' : 'datasetDocuments.segment.addKeyWord'))} + /> ) }
diff --git a/web/app/components/base/toast/index.tsx b/web/app/components/base/toast/index.tsx index ba7d8af518e51a..b9a6de9fe5ac00 100644 --- a/web/app/components/base/toast/index.tsx +++ b/web/app/components/base/toast/index.tsx @@ -21,7 +21,6 @@ export type IToastProps = { children?: ReactNode onClose?: () => void className?: string - customComponent?: ReactNode } type IToastContext = { notify: (props: IToastProps) => void @@ -36,7 +35,6 @@ const Toast = ({ message, children, className, - customComponent, }: IToastProps) => { const { close } = useToastContext() // sometimes message is react node array. Not handle it. @@ -51,7 +49,8 @@ const Toast = ({ 'top-0', 'right-0', )}> - -
-
-
{message}
- {customComponent} -
+
+
{message}
{children &&
{children}
}
- +
@@ -121,8 +117,7 @@ Toast.notify = ({ message, duration, className, - customComponent, -}: Pick) => { +}: Pick) => { const defaultDuring = (type === 'success' || type === 'info') ? 3000 : 6000 if (typeof window === 'object') { const holder = document.createElement('div') @@ -138,7 +133,7 @@ Toast.notify = ({ } }, }}> - + , ) document.body.appendChild(holder) diff --git a/web/app/components/base/tooltip/index.tsx b/web/app/components/base/tooltip/index.tsx index 65b5a99077ca14..8ec3cd8c7ab4b0 100644 --- a/web/app/components/base/tooltip/index.tsx +++ b/web/app/components/base/tooltip/index.tsx @@ -14,7 +14,6 @@ export type TooltipProps = { popupContent?: React.ReactNode children?: React.ReactNode popupClassName?: string - noDecoration?: boolean offset?: OffsetOptions needsDelay?: boolean asChild?: boolean @@ -28,7 +27,6 @@ const Tooltip: FC = ({ popupContent, children, popupClassName, - noDecoration, offset, asChild = true, needsDelay = false, @@ -98,7 +96,7 @@ const Tooltip: FC = ({ > {popupContent && (
triggerMethod === 'hover' && setHoverPopup()} diff --git a/web/app/components/billing/priority-label/index.tsx b/web/app/components/billing/priority-label/index.tsx index 6ecac4a79ea504..36338cf4a8e767 100644 --- a/web/app/components/billing/priority-label/index.tsx +++ b/web/app/components/billing/priority-label/index.tsx @@ -4,7 +4,6 @@ import { DocumentProcessingPriority, Plan, } from '../type' -import cn from '@/utils/classnames' import { useProviderContext } from '@/context/provider-context' import { ZapFast, @@ -12,11 +11,7 @@ import { } from '@/app/components/base/icons/src/vender/solid/general' import Tooltip from '@/app/components/base/tooltip' -type PriorityLabelProps = { - className?: string -} - -const PriorityLabel = ({ className }: PriorityLabelProps) => { +const PriorityLabel = () => { const { t } = useTranslation() const { plan } = useProviderContext() @@ -42,18 +37,18 @@ const PriorityLabel = ({ className }: PriorityLabelProps) => { }
}> - + { plan.type === Plan.professional && ( - + ) } { (plan.type === Plan.team || plan.type === Plan.enterprise) && ( - + ) } {t(`billing.plansCommon.priority.${priority}`)} diff --git a/web/app/components/datasets/chunk.tsx b/web/app/components/datasets/chunk.tsx deleted file mode 100644 index bf2835dbdbe183..00000000000000 --- a/web/app/components/datasets/chunk.tsx +++ /dev/null @@ -1,54 +0,0 @@ -import type { FC, PropsWithChildren } from 'react' -import { SelectionMod } from '../base/icons/src/public/knowledge' -import type { QA } from '@/models/datasets' - -export type ChunkLabelProps = { - label: string - characterCount: number -} - -export const ChunkLabel: FC = (props) => { - const { label, characterCount } = props - return
- -

- {label} - - - · - - - {`${characterCount} characters`} -

-
-} - -export type ChunkContainerProps = ChunkLabelProps & PropsWithChildren - -export const ChunkContainer: FC = (props) => { - const { label, characterCount, children } = props - return
- -
- {children} -
-
-} - -export type QAPreviewProps = { - qa: QA -} - -export const QAPreview: FC = (props) => { - const { qa } = props - return
-
- -

{qa.question}

-
-
- -

{qa.answer}

-
-
-} diff --git a/web/app/components/datasets/common/chunking-mode-label.tsx b/web/app/components/datasets/common/chunking-mode-label.tsx deleted file mode 100644 index 7c6e924009dd7b..00000000000000 --- a/web/app/components/datasets/common/chunking-mode-label.tsx +++ /dev/null @@ -1,29 +0,0 @@ -'use client' -import type { FC } from 'react' -import React from 'react' -import { useTranslation } from 'react-i18next' -import Badge from '@/app/components/base/badge' -import { GeneralType, ParentChildType } from '@/app/components/base/icons/src/public/knowledge' - -type Props = { - isGeneralMode: boolean - isQAMode: boolean -} - -const ChunkingModeLabel: FC = ({ - isGeneralMode, - isQAMode, -}) => { - const { t } = useTranslation() - const TypeIcon = isGeneralMode ? GeneralType : ParentChildType - - return ( - -
- - {isGeneralMode ? `${t('dataset.chunkingMode.general')}${isQAMode ? ' · QA' : ''}` : t('dataset.chunkingMode.parentChild')} -
- - ) -} -export default React.memo(ChunkingModeLabel) diff --git a/web/app/components/datasets/common/document-file-icon.tsx b/web/app/components/datasets/common/document-file-icon.tsx deleted file mode 100644 index 5842cbbc7c3fca..00000000000000 --- a/web/app/components/datasets/common/document-file-icon.tsx +++ /dev/null @@ -1,40 +0,0 @@ -'use client' -import type { FC } from 'react' -import React from 'react' -import FileTypeIcon from '../../base/file-uploader/file-type-icon' -import type { FileAppearanceType } from '@/app/components/base/file-uploader/types' -import { FileAppearanceTypeEnum } from '@/app/components/base/file-uploader/types' - -const extendToFileTypeMap: { [key: string]: FileAppearanceType } = { - pdf: FileAppearanceTypeEnum.pdf, - json: FileAppearanceTypeEnum.document, - html: FileAppearanceTypeEnum.document, - txt: FileAppearanceTypeEnum.document, - markdown: FileAppearanceTypeEnum.markdown, - md: FileAppearanceTypeEnum.markdown, - xlsx: FileAppearanceTypeEnum.excel, - xls: FileAppearanceTypeEnum.excel, - csv: FileAppearanceTypeEnum.excel, - doc: FileAppearanceTypeEnum.word, - docx: FileAppearanceTypeEnum.word, -} - -type Props = { - extension?: string - name?: string - size?: 'sm' | 'lg' | 'md' - className?: string -} - -const DocumentFileIcon: FC = ({ - extension, - name, - size = 'md', - className, -}) => { - const localExtension = extension?.toLowerCase() || name?.split('.')?.pop()?.toLowerCase() - return ( - - ) -} -export default React.memo(DocumentFileIcon) diff --git a/web/app/components/datasets/common/document-picker/document-list.tsx b/web/app/components/datasets/common/document-picker/document-list.tsx deleted file mode 100644 index 3e320d75073416..00000000000000 --- a/web/app/components/datasets/common/document-picker/document-list.tsx +++ /dev/null @@ -1,42 +0,0 @@ -'use client' -import type { FC } from 'react' -import React, { useCallback } from 'react' -import FileIcon from '../document-file-icon' -import cn from '@/utils/classnames' -import type { DocumentItem } from '@/models/datasets' - -type Props = { - className?: string - list: DocumentItem[] - onChange: (value: DocumentItem) => void -} - -const DocumentList: FC = ({ - className, - list, - onChange, -}) => { - const handleChange = useCallback((item: DocumentItem) => { - return () => onChange(item) - }, [onChange]) - - return ( -
- {list.map((item) => { - const { id, name, extension } = item - return ( -
- -
{name}
-
- ) - })} -
- ) -} - -export default React.memo(DocumentList) diff --git a/web/app/components/datasets/common/document-picker/index.tsx b/web/app/components/datasets/common/document-picker/index.tsx deleted file mode 100644 index 30690fca007cef..00000000000000 --- a/web/app/components/datasets/common/document-picker/index.tsx +++ /dev/null @@ -1,118 +0,0 @@ -'use client' -import type { FC } from 'react' -import React, { useCallback, useState } from 'react' -import { useBoolean } from 'ahooks' -import { RiArrowDownSLine } from '@remixicon/react' -import { useTranslation } from 'react-i18next' -import FileIcon from '../document-file-icon' -import DocumentList from './document-list' -import type { DocumentItem, ParentMode, SimpleDocumentDetail } from '@/models/datasets' -import { ProcessMode } from '@/models/datasets' -import { - PortalToFollowElem, - PortalToFollowElemContent, - PortalToFollowElemTrigger, -} from '@/app/components/base/portal-to-follow-elem' -import cn from '@/utils/classnames' -import SearchInput from '@/app/components/base/search-input' -import { GeneralType, ParentChildType } from '@/app/components/base/icons/src/public/knowledge' -import { useDocumentList } from '@/service/knowledge/use-document' -import Loading from '@/app/components/base/loading' - -type Props = { - datasetId: string - value: { - name?: string - extension?: string - processMode?: ProcessMode - parentMode?: ParentMode - } - onChange: (value: SimpleDocumentDetail) => void -} - -const DocumentPicker: FC = ({ - datasetId, - value, - onChange, -}) => { - const { t } = useTranslation() - const { - name, - extension, - processMode, - parentMode, - } = value - const [query, setQuery] = useState('') - - const { data } = useDocumentList({ - datasetId, - query: { - keyword: query, - page: 1, - limit: 20, - }, - }) - const documentsList = data?.data - const isParentChild = processMode === ProcessMode.parentChild - const TypeIcon = isParentChild ? ParentChildType : GeneralType - - const [open, { - set: setOpen, - toggle: togglePopup, - }] = useBoolean(false) - const ArrowIcon = RiArrowDownSLine - - const handleChange = useCallback(({ id }: DocumentItem) => { - onChange(documentsList?.find(item => item.id === id) as SimpleDocumentDetail) - setOpen(false) - }, [documentsList, onChange, setOpen]) - - return ( - - -
- -
-
- {name || '--'} - -
-
- - - {isParentChild ? t('dataset.chunkingMode.parentChild') : t('dataset.chunkingMode.general')} - {isParentChild && ` · ${!parentMode ? '--' : parentMode === 'paragraph' ? t('dataset.parentMode.paragraph') : t('dataset.parentMode.fullDoc')}`} - -
-
-
- - -
- - {documentsList - ? ( - ({ - id: d.id, - name: d.name, - extension: d.data_source_detail_dict?.upload_file?.extension || '', - }))} - onChange={handleChange} - /> - ) - : (
- -
)} -
- - - - ) -} -export default React.memo(DocumentPicker) diff --git a/web/app/components/datasets/common/document-picker/preview-document-picker.tsx b/web/app/components/datasets/common/document-picker/preview-document-picker.tsx deleted file mode 100644 index 2a35b75471cde8..00000000000000 --- a/web/app/components/datasets/common/document-picker/preview-document-picker.tsx +++ /dev/null @@ -1,82 +0,0 @@ -'use client' -import type { FC } from 'react' -import React, { useCallback } from 'react' -import { useBoolean } from 'ahooks' -import { RiArrowDownSLine } from '@remixicon/react' -import { useTranslation } from 'react-i18next' -import FileIcon from '../document-file-icon' -import DocumentList from './document-list' -import { - PortalToFollowElem, - PortalToFollowElemContent, - PortalToFollowElemTrigger, -} from '@/app/components/base/portal-to-follow-elem' -import cn from '@/utils/classnames' -import Loading from '@/app/components/base/loading' -import type { DocumentItem } from '@/models/datasets' - -type Props = { - className?: string - value: DocumentItem - files: DocumentItem[] - onChange: (value: DocumentItem) => void -} - -const PreviewDocumentPicker: FC = ({ - className, - value, - files, - onChange, -}) => { - const { t } = useTranslation() - const { name, extension } = value - - const [open, { - set: setOpen, - toggle: togglePopup, - }] = useBoolean(false) - const ArrowIcon = RiArrowDownSLine - - const handleChange = useCallback((item: DocumentItem) => { - onChange(item) - setOpen(false) - }, [onChange, setOpen]) - - return ( - - -
- -
-
- {name || '--'} - -
-
-
- - -
- {files?.length > 1 &&
{t('dataset.preprocessDocument', { num: files.length })}
} - {files?.length > 0 - ? ( - - ) - : (
- -
)} -
- - - - ) -} -export default React.memo(PreviewDocumentPicker) diff --git a/web/app/components/datasets/common/document-status-with-action/auto-disabled-document.tsx b/web/app/components/datasets/common/document-status-with-action/auto-disabled-document.tsx deleted file mode 100644 index b687c004e5c5cd..00000000000000 --- a/web/app/components/datasets/common/document-status-with-action/auto-disabled-document.tsx +++ /dev/null @@ -1,38 +0,0 @@ -'use client' -import type { FC } from 'react' -import React, { useCallback } from 'react' -import { useTranslation } from 'react-i18next' -import StatusWithAction from './status-with-action' -import { useAutoDisabledDocuments, useDocumentEnable, useInvalidDisabledDocument } from '@/service/knowledge/use-document' -import Toast from '@/app/components/base/toast' -type Props = { - datasetId: string -} - -const AutoDisabledDocument: FC = ({ - datasetId, -}) => { - const { t } = useTranslation() - const { data, isLoading } = useAutoDisabledDocuments(datasetId) - const invalidDisabledDocument = useInvalidDisabledDocument() - const documentIds = data?.document_ids - const hasDisabledDocument = documentIds && documentIds.length > 0 - const { mutateAsync: enableDocument } = useDocumentEnable() - const handleEnableDocuments = useCallback(async () => { - await enableDocument({ datasetId, documentIds }) - invalidDisabledDocument() - Toast.notify({ type: 'success', message: t('common.actionMsg.modifiedSuccessfully') }) - }, []) - if (!hasDisabledDocument || isLoading) - return null - - return ( - - ) -} -export default React.memo(AutoDisabledDocument) diff --git a/web/app/components/datasets/common/document-status-with-action/index-failed.tsx b/web/app/components/datasets/common/document-status-with-action/index-failed.tsx deleted file mode 100644 index 37311768b95756..00000000000000 --- a/web/app/components/datasets/common/document-status-with-action/index-failed.tsx +++ /dev/null @@ -1,69 +0,0 @@ -'use client' -import type { FC } from 'react' -import React, { useEffect, useReducer } from 'react' -import { useTranslation } from 'react-i18next' -import useSWR from 'swr' -import StatusWithAction from './status-with-action' -import { getErrorDocs, retryErrorDocs } from '@/service/datasets' -import type { IndexingStatusResponse } from '@/models/datasets' - -type Props = { - datasetId: string -} -type IIndexState = { - value: string -} -type ActionType = 'retry' | 'success' | 'error' - -type IAction = { - type: ActionType -} -const indexStateReducer = (state: IIndexState, action: IAction) => { - const actionMap = { - retry: 'retry', - success: 'success', - error: 'error', - } - - return { - ...state, - value: actionMap[action.type] || state.value, - } -} - -const RetryButton: FC = ({ datasetId }) => { - const { t } = useTranslation() - const [indexState, dispatch] = useReducer(indexStateReducer, { value: 'success' }) - const { data: errorDocs, isLoading } = useSWR({ datasetId }, getErrorDocs) - - const onRetryErrorDocs = async () => { - dispatch({ type: 'retry' }) - const document_ids = errorDocs?.data.map((doc: IndexingStatusResponse) => doc.id) || [] - const res = await retryErrorDocs({ datasetId, document_ids }) - if (res.result === 'success') - dispatch({ type: 'success' }) - else - dispatch({ type: 'error' }) - } - - useEffect(() => { - if (errorDocs?.total === 0) - dispatch({ type: 'success' }) - else - dispatch({ type: 'error' }) - }, [errorDocs?.total]) - - if (isLoading || indexState.value === 'success') - return null - - return ( - { }} - /> - ) -} -export default RetryButton diff --git a/web/app/components/datasets/common/document-status-with-action/status-with-action.tsx b/web/app/components/datasets/common/document-status-with-action/status-with-action.tsx deleted file mode 100644 index a8da9bf6cc2a8b..00000000000000 --- a/web/app/components/datasets/common/document-status-with-action/status-with-action.tsx +++ /dev/null @@ -1,65 +0,0 @@ -'use client' -import { RiAlertFill, RiCheckboxCircleFill, RiErrorWarningFill, RiInformation2Fill } from '@remixicon/react' -import type { FC } from 'react' -import React from 'react' -import cn from '@/utils/classnames' -import Divider from '@/app/components/base/divider' - -type Status = 'success' | 'error' | 'warning' | 'info' -type Props = { - type?: Status - description: string - actionText: string - onAction: () => void - disabled?: boolean -} - -const IconMap = { - success: { - Icon: RiCheckboxCircleFill, - color: 'text-text-success', - }, - error: { - Icon: RiErrorWarningFill, - color: 'text-text-destructive', - }, - warning: { - Icon: RiAlertFill, - color: 'text-text-warning-secondary', - }, - info: { - Icon: RiInformation2Fill, - color: 'text-text-accent', - }, -} - -const getIcon = (type: Status) => { - return IconMap[type] -} - -const StatusAction: FC = ({ - type = 'info', - description, - actionText, - onAction, - disabled, -}) => { - const { Icon, color } = getIcon(type) - return ( -
-
-
- -
{description}
- -
{actionText}
-
-
- ) -} -export default React.memo(StatusAction) diff --git a/web/app/components/datasets/common/economical-retrieval-method-config/index.tsx b/web/app/components/datasets/common/economical-retrieval-method-config/index.tsx index 9236858ae4c906..f3da67b92cc5e9 100644 --- a/web/app/components/datasets/common/economical-retrieval-method-config/index.tsx +++ b/web/app/components/datasets/common/economical-retrieval-method-config/index.tsx @@ -2,11 +2,10 @@ import type { FC } from 'react' import React from 'react' import { useTranslation } from 'react-i18next' -import Image from 'next/image' import RetrievalParamConfig from '../retrieval-param-config' -import { OptionCard } from '../../create/step-two/option-card' -import { retrievalIcon } from '../../create/icons' import { RETRIEVE_METHOD } from '@/types/app' +import RadioCard from '@/app/components/base/radio-card' +import { HighPriority } from '@/app/components/base/icons/src/vender/solid/arrows' import type { RetrievalConfig } from '@/types/app' type Props = { @@ -22,17 +21,19 @@ const EconomicalRetrievalMethodConfig: FC = ({ return (
- } + } title={t('dataset.retrieval.invertedIndex.title')} - description={t('dataset.retrieval.invertedIndex.description')} isActive - activeHeaderClassName='bg-dataset-option-card-purple-gradient' - > - - + description={t('dataset.retrieval.invertedIndex.description')} + noRadio + chosenConfig={ + + } + />
) } diff --git a/web/app/components/datasets/common/retrieval-method-config/index.tsx b/web/app/components/datasets/common/retrieval-method-config/index.tsx index 9ab157571b5b66..20d93568addbb7 100644 --- a/web/app/components/datasets/common/retrieval-method-config/index.tsx +++ b/web/app/components/datasets/common/retrieval-method-config/index.tsx @@ -2,13 +2,12 @@ import type { FC } from 'react' import React from 'react' import { useTranslation } from 'react-i18next' -import Image from 'next/image' import RetrievalParamConfig from '../retrieval-param-config' -import { OptionCard } from '../../create/step-two/option-card' -import Effect from '../../create/assets/option-card-effect-purple.svg' -import { retrievalIcon } from '../../create/icons' import type { RetrievalConfig } from '@/types/app' import { RETRIEVE_METHOD } from '@/types/app' +import RadioCard from '@/app/components/base/radio-card' +import { PatternRecognition, Semantic } from '@/app/components/base/icons/src/vender/solid/development' +import { FileSearch02 } from '@/app/components/base/icons/src/vender/solid/files' import { useProviderContext } from '@/context/provider-context' import { useDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' @@ -17,7 +16,6 @@ import { RerankingModeEnum, WeightedScoreEnum, } from '@/models/datasets' -import Badge from '@/app/components/base/badge' type Props = { value: RetrievalConfig @@ -58,72 +56,67 @@ const RetrievalMethodConfig: FC = ({ return (
{supportRetrievalMethods.includes(RETRIEVE_METHOD.semantic) && ( - } + } title={t('dataset.retrieval.semantic_search.title')} description={t('dataset.retrieval.semantic_search.description')} - isActive={ - value.search_method === RETRIEVE_METHOD.semantic - } - onSwitched={() => onChange({ + isChosen={value.search_method === RETRIEVE_METHOD.semantic} + onChosen={() => onChange({ ...value, search_method: RETRIEVE_METHOD.semantic, })} - effectImg={Effect.src} - activeHeaderClassName='bg-dataset-option-card-purple-gradient' - > - - + chosenConfig={ + + } + /> )} {supportRetrievalMethods.includes(RETRIEVE_METHOD.semantic) && ( - } + } title={t('dataset.retrieval.full_text_search.title')} description={t('dataset.retrieval.full_text_search.description')} - isActive={ - value.search_method === RETRIEVE_METHOD.fullText - } - onSwitched={() => onChange({ + isChosen={value.search_method === RETRIEVE_METHOD.fullText} + onChosen={() => onChange({ ...value, search_method: RETRIEVE_METHOD.fullText, })} - effectImg={Effect.src} - activeHeaderClassName='bg-dataset-option-card-purple-gradient' - > - - + chosenConfig={ + + } + /> )} {supportRetrievalMethods.includes(RETRIEVE_METHOD.semantic) && ( - } + } title={
{t('dataset.retrieval.hybrid_search.title')}
- +
{t('dataset.retrieval.hybrid_search.recommend')}
} - description={t('dataset.retrieval.hybrid_search.description')} isActive={ - value.search_method === RETRIEVE_METHOD.hybrid - } - onSwitched={() => onChange({ + description={t('dataset.retrieval.hybrid_search.description')} + isChosen={value.search_method === RETRIEVE_METHOD.hybrid} + onChosen={() => onChange({ ...value, search_method: RETRIEVE_METHOD.hybrid, reranking_enable: true, })} - effectImg={Effect.src} - activeHeaderClassName='bg-dataset-option-card-purple-gradient' - > - - + chosenConfig={ + + } + /> )}
) diff --git a/web/app/components/datasets/common/retrieval-method-info/index.tsx b/web/app/components/datasets/common/retrieval-method-info/index.tsx index fc3020d4a965a2..7d9b999c53941f 100644 --- a/web/app/components/datasets/common/retrieval-method-info/index.tsx +++ b/web/app/components/datasets/common/retrieval-method-info/index.tsx @@ -2,11 +2,12 @@ import type { FC } from 'react' import React from 'react' import { useTranslation } from 'react-i18next' -import Image from 'next/image' -import { retrievalIcon } from '../../create/icons' import type { RetrievalConfig } from '@/types/app' import { RETRIEVE_METHOD } from '@/types/app' import RadioCard from '@/app/components/base/radio-card' +import { HighPriority } from '@/app/components/base/icons/src/vender/solid/arrows' +import { PatternRecognition, Semantic } from '@/app/components/base/icons/src/vender/solid/development' +import { FileSearch02 } from '@/app/components/base/icons/src/vender/solid/files' type Props = { value: RetrievalConfig @@ -14,12 +15,11 @@ type Props = { export const getIcon = (type: RETRIEVE_METHOD) => { return ({ - [RETRIEVE_METHOD.semantic]: retrievalIcon.vector, - [RETRIEVE_METHOD.fullText]: retrievalIcon.fullText, - [RETRIEVE_METHOD.hybrid]: retrievalIcon.hybrid, - [RETRIEVE_METHOD.invertedIndex]: retrievalIcon.vector, - [RETRIEVE_METHOD.keywordSearch]: retrievalIcon.vector, - })[type] || retrievalIcon.vector + [RETRIEVE_METHOD.semantic]: Semantic, + [RETRIEVE_METHOD.fullText]: FileSearch02, + [RETRIEVE_METHOD.hybrid]: PatternRecognition, + [RETRIEVE_METHOD.invertedIndex]: HighPriority, + })[type] || FileSearch02 } const EconomicalRetrievalMethodConfig: FC = ({ @@ -28,11 +28,11 @@ const EconomicalRetrievalMethodConfig: FC = ({ }) => { const { t } = useTranslation() const type = value.search_method - const icon = + const Icon = getIcon(type) return (
} title={t(`dataset.retrieval.${type}.title`)} description={t(`dataset.retrieval.${type}.description`)} noRadio diff --git a/web/app/components/datasets/common/retrieval-param-config/index.tsx b/web/app/components/datasets/common/retrieval-param-config/index.tsx index 5136ac1659159d..9d48d56a8dc511 100644 --- a/web/app/components/datasets/common/retrieval-param-config/index.tsx +++ b/web/app/components/datasets/common/retrieval-param-config/index.tsx @@ -3,9 +3,6 @@ import type { FC } from 'react' import React, { useCallback } from 'react' import { useTranslation } from 'react-i18next' -import Image from 'next/image' -import ProgressIndicator from '../../create/assets/progress-indicator.svg' -import Reranking from '../../create/assets/rerank.svg' import cn from '@/utils/classnames' import TopKItem from '@/app/components/base/param-item/top-k-item' import ScoreThresholdItem from '@/app/components/base/param-item/score-threshold-item' @@ -23,7 +20,6 @@ import { } from '@/models/datasets' import WeightedScore from '@/app/components/app/configuration/dataset-config/params-config/weighted-score' import Toast from '@/app/components/base/toast' -import RadioCard from '@/app/components/base/radio-card' type Props = { type: RETRIEVE_METHOD @@ -120,7 +116,7 @@ const RetrievalParamConfig: FC = ({
{!isEconomical && !isHybridSearch && (
-
+
{canToggleRerankModalEnable && (
= ({
)}
- {t('common.modelProvider.rerankModel.key')} + {t('common.modelProvider.rerankModel.key')} {t('common.modelProvider.rerankModel.tip')}
@@ -167,7 +163,7 @@ const RetrievalParamConfig: FC = ({ )} { !isHybridSearch && ( -
+
= ({ { isHybridSearch && ( <> -
+
{ rerankingModeOptions.map(option => ( - handleChangeRerankMode(option.value)} - icon={} - title={option.label} - description={option.tips} - className='flex-1' - /> + className={cn( + 'flex items-center justify-center mb-4 w-[calc((100%-8px)/2)] h-8 rounded-lg border border-components-option-card-option-border bg-components-option-card-option-bg cursor-pointer system-sm-medium text-text-secondary', + value.reranking_mode === RerankingModeEnum.WeightedScore && option.value === RerankingModeEnum.WeightedScore && 'border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary', + value.reranking_mode !== RerankingModeEnum.WeightedScore && option.value !== RerankingModeEnum.WeightedScore && 'border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary', + )} + onClick={() => handleChangeRerankMode(option.value)} + > +
{option.label}
+ {option.tips}
} + triggerClassName='ml-0.5 w-3.5 h-3.5' + /> +
)) }
diff --git a/web/app/components/datasets/create/assets/family-mod.svg b/web/app/components/datasets/create/assets/family-mod.svg deleted file mode 100644 index b1c4e6f566e54f..00000000000000 --- a/web/app/components/datasets/create/assets/family-mod.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/web/app/components/datasets/create/assets/file-list-3-fill.svg b/web/app/components/datasets/create/assets/file-list-3-fill.svg deleted file mode 100644 index a4e6c4da9783e8..00000000000000 --- a/web/app/components/datasets/create/assets/file-list-3-fill.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/web/app/components/datasets/create/assets/gold.svg b/web/app/components/datasets/create/assets/gold.svg deleted file mode 100644 index b48ac0eae5de02..00000000000000 --- a/web/app/components/datasets/create/assets/gold.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/web/app/components/datasets/create/assets/note-mod.svg b/web/app/components/datasets/create/assets/note-mod.svg deleted file mode 100644 index b9e81f6bd533b5..00000000000000 --- a/web/app/components/datasets/create/assets/note-mod.svg +++ /dev/null @@ -1,5 +0,0 @@ - - - - - diff --git a/web/app/components/datasets/create/assets/option-card-effect-blue.svg b/web/app/components/datasets/create/assets/option-card-effect-blue.svg deleted file mode 100644 index 00a8afad8b1b23..00000000000000 --- a/web/app/components/datasets/create/assets/option-card-effect-blue.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - diff --git a/web/app/components/datasets/create/assets/option-card-effect-orange.svg b/web/app/components/datasets/create/assets/option-card-effect-orange.svg deleted file mode 100644 index d833764f0cba63..00000000000000 --- a/web/app/components/datasets/create/assets/option-card-effect-orange.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - diff --git a/web/app/components/datasets/create/assets/option-card-effect-purple.svg b/web/app/components/datasets/create/assets/option-card-effect-purple.svg deleted file mode 100644 index a7857f8e570f05..00000000000000 --- a/web/app/components/datasets/create/assets/option-card-effect-purple.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - diff --git a/web/app/components/datasets/create/assets/pattern-recognition-mod.svg b/web/app/components/datasets/create/assets/pattern-recognition-mod.svg deleted file mode 100644 index 1083e888ed6820..00000000000000 --- a/web/app/components/datasets/create/assets/pattern-recognition-mod.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/web/app/components/datasets/create/assets/piggy-bank-mod.svg b/web/app/components/datasets/create/assets/piggy-bank-mod.svg deleted file mode 100644 index b1120ad9a9c249..00000000000000 --- a/web/app/components/datasets/create/assets/piggy-bank-mod.svg +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/web/app/components/datasets/create/assets/progress-indicator.svg b/web/app/components/datasets/create/assets/progress-indicator.svg deleted file mode 100644 index 3c997136360595..00000000000000 --- a/web/app/components/datasets/create/assets/progress-indicator.svg +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - diff --git a/web/app/components/datasets/create/assets/rerank.svg b/web/app/components/datasets/create/assets/rerank.svg deleted file mode 100644 index 409b52e6e23804..00000000000000 --- a/web/app/components/datasets/create/assets/rerank.svg +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/web/app/components/datasets/create/assets/research-mod.svg b/web/app/components/datasets/create/assets/research-mod.svg deleted file mode 100644 index 1f0bb3423351a1..00000000000000 --- a/web/app/components/datasets/create/assets/research-mod.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/web/app/components/datasets/create/assets/selection-mod.svg b/web/app/components/datasets/create/assets/selection-mod.svg deleted file mode 100644 index 2d0dd3b5f74a4f..00000000000000 --- a/web/app/components/datasets/create/assets/selection-mod.svg +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/web/app/components/datasets/create/assets/setting-gear-mod.svg b/web/app/components/datasets/create/assets/setting-gear-mod.svg deleted file mode 100644 index c782caade88e15..00000000000000 --- a/web/app/components/datasets/create/assets/setting-gear-mod.svg +++ /dev/null @@ -1,4 +0,0 @@ - - - - \ No newline at end of file diff --git a/web/app/components/datasets/create/embedding-process/index.module.css b/web/app/components/datasets/create/embedding-process/index.module.css index f2ab4d85a27210..1ebb006b543ac2 100644 --- a/web/app/components/datasets/create/embedding-process/index.module.css +++ b/web/app/components/datasets/create/embedding-process/index.module.css @@ -14,7 +14,24 @@ border-radius: 6px; overflow: hidden; } - +.sourceItem.error { + background: #FEE4E2; +} +.sourceItem.success { + background: #D1FADF; +} +.progressbar { + position: absolute; + top: 0; + left: 0; + height: 100%; + background-color: #B2CCFF; +} +.sourceItem .info { + display: flex; + align-items: center; + z-index: 1; +} .sourceItem .info .name { font-weight: 500; font-size: 12px; @@ -38,6 +55,13 @@ color: #05603A; } + +.cost { + @apply flex justify-between items-center text-xs text-gray-700; +} +.embeddingStatus { + @apply flex items-center justify-between text-gray-900 font-medium text-sm mr-2; +} .commonIcon { @apply w-3 h-3 mr-1 inline-block align-middle; } @@ -57,33 +81,35 @@ @apply text-xs font-medium; } -.unknownFileIcon { +.fileIcon { + @apply w-4 h-4 mr-1 bg-center bg-no-repeat; background-image: url(../assets/unknown.svg); + background-size: 16px; } -.csv { +.fileIcon.csv { background-image: url(../assets/csv.svg); } -.docx { +.fileIcon.docx { background-image: url(../assets/docx.svg); } -.xlsx, -.xls { +.fileIcon.xlsx, +.fileIcon.xls { background-image: url(../assets/xlsx.svg); } -.pdf { +.fileIcon.pdf { background-image: url(../assets/pdf.svg); } -.html, -.htm { +.fileIcon.html, +.fileIcon.htm { background-image: url(../assets/html.svg); } -.md, -.markdown { +.fileIcon.md, +.fileIcon.markdown { background-image: url(../assets/md.svg); } -.txt { +.fileIcon.txt { background-image: url(../assets/txt.svg); } -.json { +.fileIcon.json { background-image: url(../assets/json.svg); } diff --git a/web/app/components/datasets/create/embedding-process/index.tsx b/web/app/components/datasets/create/embedding-process/index.tsx index 201333ffce4cbe..7786582085c16d 100644 --- a/web/app/components/datasets/create/embedding-process/index.tsx +++ b/web/app/components/datasets/create/embedding-process/index.tsx @@ -6,44 +6,32 @@ import { useTranslation } from 'react-i18next' import { omit } from 'lodash-es' import { ArrowRightIcon } from '@heroicons/react/24/solid' import { - RiCheckboxCircleFill, RiErrorWarningFill, - RiLoader2Fill, - RiTerminalBoxLine, } from '@remixicon/react' -import Image from 'next/image' -import { indexMethodIcon, retrievalIcon } from '../icons' -import { IndexingType } from '../step-two' -import DocumentFileIcon from '../../common/document-file-icon' +import s from './index.module.css' import cn from '@/utils/classnames' import { FieldInfo } from '@/app/components/datasets/documents/detail/metadata' import Button from '@/app/components/base/button' import type { FullDocumentDetail, IndexingStatusResponse, ProcessRuleResponse } from '@/models/datasets' import { fetchIndexingStatusBatch as doFetchIndexingStatus, fetchProcessRule } from '@/service/datasets' -import { DataSourceType, ProcessMode } from '@/models/datasets' +import { DataSourceType } from '@/models/datasets' import NotionIcon from '@/app/components/base/notion-icon' import PriorityLabel from '@/app/components/billing/priority-label' import { Plan } from '@/app/components/billing/type' import { ZapFast } from '@/app/components/base/icons/src/vender/solid/general' import UpgradeBtn from '@/app/components/billing/upgrade-btn' import { useProviderContext } from '@/context/provider-context' -import { sleep } from '@/utils' -import { RETRIEVE_METHOD } from '@/types/app' import Tooltip from '@/app/components/base/tooltip' +import { sleep } from '@/utils' type Props = { datasetId: string batchId: string documents?: FullDocumentDetail[] indexingType?: string - retrievalMethod?: string } -const RuleDetail: FC<{ - sourceData?: ProcessRuleResponse - indexingType?: string - retrievalMethod?: string -}> = ({ sourceData, indexingType, retrievalMethod }) => { +const RuleDetail: FC<{ sourceData?: ProcessRuleResponse }> = ({ sourceData }) => { const { t } = useTranslation() const segmentationRuleMap = { @@ -63,47 +51,29 @@ const RuleDetail: FC<{ return t('datasetCreation.stepTwo.removeStopwords') } - const isNumber = (value: unknown) => { - return typeof value === 'number' - } - const getValue = useCallback((field: string) => { let value: string | number | undefined = '-' - const maxTokens = isNumber(sourceData?.rules?.segmentation?.max_tokens) - ? sourceData.rules.segmentation.max_tokens - : value - const childMaxTokens = isNumber(sourceData?.rules?.subchunk_segmentation?.max_tokens) - ? sourceData.rules.subchunk_segmentation.max_tokens - : value switch (field) { case 'mode': - value = !sourceData?.mode - ? value - : sourceData.mode === ProcessMode.general - ? (t('datasetDocuments.embedding.custom') as string) - : `${t('datasetDocuments.embedding.hierarchical')} · ${sourceData?.rules?.parent_mode === 'paragraph' - ? t('dataset.parentMode.paragraph') - : t('dataset.parentMode.fullDoc')}` + value = sourceData?.mode === 'automatic' ? (t('datasetDocuments.embedding.automatic') as string) : (t('datasetDocuments.embedding.custom') as string) break case 'segmentLength': - value = !sourceData?.mode - ? value - : sourceData.mode === ProcessMode.general - ? maxTokens - : `${t('datasetDocuments.embedding.parentMaxTokens')} ${maxTokens}; ${t('datasetDocuments.embedding.childMaxTokens')} ${childMaxTokens}` + value = sourceData?.rules?.segmentation?.max_tokens break default: - value = !sourceData?.mode - ? value - : sourceData?.rules?.pre_processing_rules?.filter(rule => - rule.enabled).map(rule => getRuleName(rule.id)).join(',') + value = sourceData?.mode === 'automatic' + ? (t('datasetDocuments.embedding.automatic') as string) + // eslint-disable-next-line array-callback-return + : sourceData?.rules?.pre_processing_rules?.map((rule) => { + if (rule.enabled) + return getRuleName(rule.id) + }).filter(Boolean).join(';') break } return value - // eslint-disable-next-line react-hooks/exhaustive-deps }, [sourceData]) - return
+ return
{Object.keys(segmentationRuleMap).map((field) => { return })} - - } - /> - - } - />
} -const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], indexingType, retrievalMethod }) => { +const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], indexingType }) => { const { t } = useTranslation() const { enableBilling, plan } = useProviderContext() @@ -190,7 +127,6 @@ const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], index } useEffect(() => { - setIsStopQuery(false) startQueryStatus() return () => { stopQueryStatus() @@ -210,9 +146,6 @@ const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], index const navToDocumentList = () => { router.push(`/datasets/${datasetId}/documents`) } - const navToApiDocs = () => { - router.push('/datasets?category=api') - } const isEmbedding = useMemo(() => { return indexingStatusBatchDetail.some(indexingStatusDetail => ['indexing', 'splitting', 'parsing', 'cleaning'].includes(indexingStatusDetail?.indexing_status || '')) @@ -244,17 +177,13 @@ const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], index return doc?.data_source_info.notion_page_icon } - const isSourceEmbedding = (detail: IndexingStatusResponse) => - ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'].includes(detail.indexing_status || '') + const isSourceEmbedding = (detail: IndexingStatusResponse) => ['indexing', 'splitting', 'parsing', 'cleaning', 'waiting'].includes(detail.indexing_status || '') return ( <> -
-
- {isEmbedding &&
- - {t('datasetDocuments.embedding.processing')} -
} +
+
+ {isEmbedding && t('datasetDocuments.embedding.processing')} {isEmbeddingCompleted && t('datasetDocuments.embedding.completed')}
@@ -271,80 +200,69 @@ const EmbeddingProcess: FC = ({ datasetId, batchId, documents = [], index
) } -
+
{indexingStatusBatchDetail.map(indexingStatusDetail => (
{isSourceEmbedding(indexingStatusDetail) && ( -
+
)} -
+
{getSourceType(indexingStatusDetail.id) === DataSourceType.FILE && ( - //
- +
)} {getSourceType(indexingStatusDetail.id) === DataSourceType.NOTION && ( )} -
-
- {getSourceName(indexingStatusDetail.id)} -
- { - enableBilling && ( - - ) - } -
+
{getSourceName(indexingStatusDetail.id)}
+ { + enableBilling && ( + + ) + } +
+
{isSourceEmbedding(indexingStatusDetail) && ( -
{`${getSourcePercent(indexingStatusDetail)}%`}
+
{`${getSourcePercent(indexingStatusDetail)}%`}
)} - {indexingStatusDetail.indexing_status === 'error' && ( + {indexingStatusDetail.indexing_status === 'error' && indexingStatusDetail.error && ( + {indexingStatusDetail.error} +
+ )} > - - - +
+ Error + +
)} + {indexingStatusDetail.indexing_status === 'error' && !indexingStatusDetail.error && ( +
+ Error +
+ )} {indexingStatusDetail.indexing_status === 'completed' && ( - +
100%
)}
))}
-
- -
- + +
diff --git a/web/app/components/datasets/create/file-preview/index.module.css b/web/app/components/datasets/create/file-preview/index.module.css index 929002e1e2985c..d87522e6d0bd4c 100644 --- a/web/app/components/datasets/create/file-preview/index.module.css +++ b/web/app/components/datasets/create/file-preview/index.module.css @@ -1,6 +1,6 @@ .filePreview { @apply flex flex-col border-l border-gray-200 shrink-0; - width: 100%; + width: 528px; background-color: #fcfcfd; } @@ -48,6 +48,5 @@ } .fileContent { white-space: pre-line; - word-break: break-all; } \ No newline at end of file diff --git a/web/app/components/datasets/create/file-preview/index.tsx b/web/app/components/datasets/create/file-preview/index.tsx index cb1f1d6908c4ef..e20af64386c509 100644 --- a/web/app/components/datasets/create/file-preview/index.tsx +++ b/web/app/components/datasets/create/file-preview/index.tsx @@ -44,7 +44,7 @@ const FilePreview = ({ }, [file]) return ( -
+
{t('datasetCreation.stepOne.filePreview')} @@ -59,7 +59,7 @@ const FilePreview = ({
{loading &&
} {!loading && ( -
{previewContent}
+
{previewContent}
)}
diff --git a/web/app/components/datasets/create/file-uploader/index.module.css b/web/app/components/datasets/create/file-uploader/index.module.css index 7d29f2ef9c2b51..bf5b7dcaf5b9b7 100644 --- a/web/app/components/datasets/create/file-uploader/index.module.css +++ b/web/app/components/datasets/create/file-uploader/index.module.css @@ -1,3 +1,68 @@ +.fileUploader { + @apply mb-6; +} + +.fileUploader .title { + @apply mb-2; + font-weight: 500; + font-size: 16px; + line-height: 24px; + color: #344054; +} + +.fileUploader .tip { + font-weight: 400; + font-size: 12px; + line-height: 18px; + color: #667085; +} + +.uploader { + @apply relative box-border flex justify-center items-center mb-2 p-3; + flex-direction: column; + max-width: 640px; + min-height: 80px; + background: #F9FAFB; + border: 1px dashed #EAECF0; + border-radius: 12px; + font-weight: 400; + font-size: 14px; + line-height: 20px; + color: #667085; +} + +.uploader.dragging { + background: #F5F8FF; + border: 1px dashed #B2CCFF; +} + +.uploader .draggingCover { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; +} + +.uploader .uploadIcon { + content: ''; + display: block; + margin-right: 8px; + width: 24px; + height: 24px; + background: center no-repeat url(../assets/upload-cloud-01.svg); + background-size: contain; +} + +.uploader .browse { + @apply pl-1 cursor-pointer; + color: #155eef; +} + +.fileList { + @apply space-y-2; +} + .file { @apply box-border relative flex items-center justify-between; padding: 8px 12px 8px 8px; @@ -128,4 +193,4 @@ .file:hover .actionWrapper .remove { display: block; -} +} \ No newline at end of file diff --git a/web/app/components/datasets/create/file-uploader/index.tsx b/web/app/components/datasets/create/file-uploader/index.tsx index e42a24cfef52d3..adb4bed0d167ed 100644 --- a/web/app/components/datasets/create/file-uploader/index.tsx +++ b/web/app/components/datasets/create/file-uploader/index.tsx @@ -3,12 +3,10 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import useSWR from 'swr' -import { RiDeleteBinLine, RiUploadCloud2Line } from '@remixicon/react' -import DocumentFileIcon from '../../common/document-file-icon' +import s from './index.module.css' import cn from '@/utils/classnames' import type { CustomFile as File, FileItem } from '@/models/datasets' import { ToastContext } from '@/app/components/base/toast' -import SimplePieChart from '@/app/components/base/simple-pie-chart' import { upload } from '@/service/base' import { fetchFileUploadConfig } from '@/service/common' @@ -16,8 +14,6 @@ import { fetchSupportFileTypes } from '@/service/datasets' import I18n from '@/context/i18n' import { LanguagesSupported } from '@/i18n/language' import { IS_CE_EDITION } from '@/config' -import { useAppContext } from '@/context/app-context' -import { Theme } from '@/types/app' const FILES_NUMBER_LIMIT = 20 @@ -226,9 +222,6 @@ const FileUploader = ({ initialUpload(files.filter(isValid)) }, [isValid, initialUpload]) - const { theme } = useAppContext() - const chartColor = useMemo(() => theme === Theme.dark ? '#5289ff' : '#296dff', [theme]) - useEffect(() => { dropRef.current?.addEventListener('dragenter', handleDragEnter) dropRef.current?.addEventListener('dragover', handleDragOver) @@ -243,12 +236,12 @@ const FileUploader = ({ }, [handleDrop]) return ( -
+
{!hideUpload && ( )} -
{t('datasetCreation.stepOne.uploader.title')}
- +
{t('datasetCreation.stepOne.uploader.title')}
{!hideUpload && ( -
-
- +
+
+ {t('datasetCreation.stepOne.uploader.button')} - {supportTypes.length > 0 && ( - - )} +
-
{t('datasetCreation.stepOne.uploader.tip', { +
{t('datasetCreation.stepOne.uploader.tip', { size: fileUploadConfig.file_size_limit, supportTypes: supportTypesShowNames, })}
- {dragging &&
} + {dragging &&
}
)} -
- +
{fileList.map((fileItem, index) => (
fileItem.file?.id && onPreview(fileItem.file)} className={cn( - 'flex items-center h-12 max-w-[640px] bg-components-panel-on-panel-item-bg text-xs leading-3 text-text-tertiary border border-components-panel-border rounded-lg shadow-xs', - // 'border-state-destructive-border bg-state-destructive-hover', + s.file, + fileItem.progress < 100 && s.uploading, )} > -
- -
-
-
-
{fileItem.file.name}
-
-
- {getFileType(fileItem.file)} - · - {getFileSize(fileItem.file.size)} - {/* · - 10k characters */} -
+ {fileItem.progress < 100 && ( +
+ )} +
+
+
{fileItem.file.name}
+
{getFileSize(fileItem.file.size)}
-
- {/* - - */} +
{(fileItem.progress < 100 && fileItem.progress >= 0) && ( - //
{`${fileItem.progress}%`}
- +
{`${fileItem.progress}%`}
+ )} + {fileItem.progress === 100 && ( +
{ + e.stopPropagation() + removeFile(fileItem.fileID) + }} /> )} - { - e.stopPropagation() - removeFile(fileItem.fileID) - }}> - -
))} diff --git a/web/app/components/datasets/create/icons.ts b/web/app/components/datasets/create/icons.ts deleted file mode 100644 index 80c4b6c944778b..00000000000000 --- a/web/app/components/datasets/create/icons.ts +++ /dev/null @@ -1,16 +0,0 @@ -import GoldIcon from './assets/gold.svg' -import Piggybank from './assets/piggy-bank-mod.svg' -import Selection from './assets/selection-mod.svg' -import Research from './assets/research-mod.svg' -import PatternRecognition from './assets/pattern-recognition-mod.svg' - -export const indexMethodIcon = { - high_quality: GoldIcon, - economical: Piggybank, -} - -export const retrievalIcon = { - vector: Selection, - fullText: Research, - hybrid: PatternRecognition, -} diff --git a/web/app/components/datasets/create/index.tsx b/web/app/components/datasets/create/index.tsx index 9556b9fad5780e..98098445c7695c 100644 --- a/web/app/components/datasets/create/index.tsx +++ b/web/app/components/datasets/create/index.tsx @@ -3,10 +3,10 @@ import React, { useCallback, useEffect, useState } from 'react' import { useTranslation } from 'react-i18next' import AppUnavailable from '../../base/app-unavailable' import { ModelTypeEnum } from '../../header/account-setting/model-provider-page/declarations' +import StepsNavBar from './steps-nav-bar' import StepOne from './step-one' import StepTwo from './step-two' import StepThree from './step-three' -import { Topbar } from './top-bar' import { DataSourceType } from '@/models/datasets' import type { CrawlOptions, CrawlResultItem, DataSet, FileItem, createDocumentResponse } from '@/models/datasets' import { fetchDataSource } from '@/service/common' @@ -36,7 +36,6 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { const [dataSourceType, setDataSourceType] = useState(DataSourceType.FILE) const [step, setStep] = useState(1) const [indexingTypeCache, setIndexTypeCache] = useState('') - const [retrievalMethodCache, setRetrievalMethodCache] = useState('') const [fileList, setFiles] = useState([]) const [result, setResult] = useState() const [hasError, setHasError] = useState(false) @@ -81,9 +80,6 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { const updateResultCache = (res?: createDocumentResponse) => { setResult(res) } - const updateRetrievalMethodCache = (method: string) => { - setRetrievalMethodCache(method) - } const nextStep = useCallback(() => { setStep(step + 1) @@ -122,29 +118,33 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { return return ( -
- -
- {step === 1 && setShowAccountSettingModal({ payload: 'data-source' })} - datasetId={datasetId} - dataSourceType={dataSourceType} - dataSourceTypeDisable={!!detail?.data_source_type} - changeType={setDataSourceType} - files={fileList} - updateFile={updateFile} - updateFileList={updateFileList} - notionPages={notionPages} - updateNotionPages={updateNotionPages} - onStepChange={nextStep} - websitePages={websitePages} - updateWebsitePages={setWebsitePages} - onWebsiteCrawlProviderChange={setWebsiteCrawlProvider} - onWebsiteCrawlJobIdChange={setWebsiteCrawlJobId} - crawlOptions={crawlOptions} - onCrawlOptionsChange={setCrawlOptions} - />} +
+
+ +
+
+
+ setShowAccountSettingModal({ payload: 'data-source' })} + datasetId={datasetId} + dataSourceType={dataSourceType} + dataSourceTypeDisable={!!detail?.data_source_type} + changeType={setDataSourceType} + files={fileList} + updateFile={updateFile} + updateFileList={updateFileList} + notionPages={notionPages} + updateNotionPages={updateNotionPages} + onStepChange={nextStep} + websitePages={websitePages} + updateWebsitePages={setWebsitePages} + onWebsiteCrawlProviderChange={setWebsiteCrawlProvider} + onWebsiteCrawlJobIdChange={setWebsiteCrawlJobId} + crawlOptions={crawlOptions} + onCrawlOptionsChange={setCrawlOptions} + /> +
{(step === 2 && (!datasetId || (datasetId && !!detail))) && setShowAccountSettingModal({ payload: 'provider' })} @@ -158,7 +158,6 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { websiteCrawlJobId={websiteCrawlJobId} onStepChange={changeStep} updateIndexingTypeCache={updateIndexingTypeCache} - updateRetrievalMethodCache={updateRetrievalMethodCache} updateResultCache={updateResultCache} crawlOptions={crawlOptions} />} @@ -166,7 +165,6 @@ const DatasetUpdateForm = ({ datasetId }: DatasetUpdateFormProps) => { datasetId={datasetId} datasetName={detail?.name} indexingType={detail?.indexing_technique || indexingTypeCache} - retrievalMethod={detail?.retrieval_model_dict?.search_method || retrievalMethodCache} creationCache={result} />}
diff --git a/web/app/components/datasets/create/notion-page-preview/index.tsx b/web/app/components/datasets/create/notion-page-preview/index.tsx index f658f213e85f17..8225e56f0400e4 100644 --- a/web/app/components/datasets/create/notion-page-preview/index.tsx +++ b/web/app/components/datasets/create/notion-page-preview/index.tsx @@ -44,7 +44,7 @@ const NotionPagePreview = ({ }, [currentPage]) return ( -
+
{t('datasetCreation.stepOne.pagePreview')} @@ -64,7 +64,7 @@ const NotionPagePreview = ({
{loading &&
} {!loading && ( -
{previewContent}
+
{previewContent}
)}
diff --git a/web/app/components/datasets/create/step-one/index.module.css b/web/app/components/datasets/create/step-one/index.module.css index bb8dd9b895c9b5..4e3cf67cd6c65b 100644 --- a/web/app/components/datasets/create/step-one/index.module.css +++ b/web/app/components/datasets/create/step-one/index.module.css @@ -2,19 +2,21 @@ position: sticky; top: 0; left: 0; - padding: 42px 64px 12px 0; + padding: 42px 64px 12px; font-weight: 600; font-size: 18px; line-height: 28px; + color: #101828; } .form { position: relative; padding: 12px 64px; + background-color: #fff; } .dataSourceItem { - @apply box-border relative grow shrink-0 flex items-center p-3 h-14 bg-white rounded-xl cursor-pointer; + @apply box-border relative shrink-0 flex items-center mr-3 p-3 h-14 bg-white rounded-xl cursor-pointer; border: 0.5px solid #EAECF0; box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05); font-weight: 500; @@ -22,32 +24,27 @@ line-height: 20px; color: #101828; } - .dataSourceItem:hover { background-color: #f5f8ff; border: 0.5px solid #B2CCFF; box-shadow: 0px 12px 16px -4px rgba(16, 24, 40, 0.08), 0px 4px 6px -2px rgba(16, 24, 40, 0.03); } - .dataSourceItem.active { background-color: #f5f8ff; border: 1.5px solid #528BFF; box-shadow: 0px 1px 3px rgba(16, 24, 40, 0.1), 0px 1px 2px rgba(16, 24, 40, 0.06); } - .dataSourceItem.disabled { background-color: #f9fafb; border: 0.5px solid #EAECF0; box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05); cursor: default; } - .dataSourceItem.disabled:hover { background-color: #f9fafb; border: 0.5px solid #EAECF0; box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05); } - .comingTag { @apply flex justify-center items-center bg-white; position: absolute; @@ -62,7 +59,6 @@ line-height: 18px; color: #444CE7; } - .datasetIcon { @apply flex mr-2 w-8 h-8 rounded-lg bg-center bg-no-repeat; background-color: #F5FAFF; @@ -70,18 +66,15 @@ background-size: 16px; border: 0.5px solid #D1E9FF; } - .dataSourceItem:active .datasetIcon, .dataSourceItem:hover .datasetIcon { background-color: #F5F8FF; border: 0.5px solid #E0EAFF; } - .datasetIcon.notion { background-image: url(../assets/notion.svg); background-size: 20px; } - .datasetIcon.web { background-image: url(../assets/web.svg); } @@ -97,12 +90,29 @@ background-color: #eaecf0; } +.OtherCreationOption { + @apply flex items-center cursor-pointer; + font-weight: 500; + font-size: 13px; + line-height: 18px; + color: #155EEF; +} +.OtherCreationOption::before { + content: ''; + display: block; + margin-right: 4px; + width: 16px; + height: 16px; + background: center no-repeat url(../assets/folder-plus.svg); + background-size: contain; +} + .notionConnectionTip { display: flex; flex-direction: column; align-items: flex-start; padding: 24px; - width: 640px; + max-width: 640px; background: #F9FAFB; border-radius: 16px; } @@ -128,7 +138,6 @@ line-height: 24px; color: #374151; } - .notionConnectionTip .title::after { content: ''; position: absolute; @@ -139,7 +148,6 @@ background: center no-repeat url(../assets/Icon-3-dots.svg); background-size: contain; } - .notionConnectionTip .tip { margin-bottom: 20px; font-style: normal; @@ -147,4 +155,4 @@ font-size: 13px; line-height: 18px; color: #6B7280; -} \ No newline at end of file +} diff --git a/web/app/components/datasets/create/step-one/index.tsx b/web/app/components/datasets/create/step-one/index.tsx index 2cca003b397207..643932e9ae21d5 100644 --- a/web/app/components/datasets/create/step-one/index.tsx +++ b/web/app/components/datasets/create/step-one/index.tsx @@ -1,7 +1,6 @@ 'use client' import React, { useMemo, useState } from 'react' import { useTranslation } from 'react-i18next' -import { RiArrowRightLine, RiFolder6Line } from '@remixicon/react' import FilePreview from '../file-preview' import FileUploader from '../file-uploader' import NotionPagePreview from '../notion-page-preview' @@ -18,7 +17,6 @@ import { NotionPageSelector } from '@/app/components/base/notion-page-selector' import { useDatasetDetailContext } from '@/context/dataset-detail' import { useProviderContext } from '@/context/provider-context' import VectorSpaceFull from '@/app/components/billing/vector-space-full' -import classNames from '@/utils/classnames' type IStepOneProps = { datasetId?: string @@ -122,174 +120,143 @@ const StepOne = ({ return true if (isShowVectorSpaceFull) return true - return false - }, [files, isShowVectorSpaceFull]) + return false + }, [files]) return (
-
-
-
- { - shouldShowDataSourceTypeList && ( -
{t('datasetCreation.steps.one')}
- ) - } - { - shouldShowDataSourceTypeList && ( -
-
{ - if (dataSourceTypeDisable) - return - changeType(DataSourceType.FILE) - hideFilePreview() - hideNotionPagePreview() - }} - > - - {t('datasetCreation.stepOne.dataSourceType.file')} -
-
{ - if (dataSourceTypeDisable) - return - changeType(DataSourceType.NOTION) - hideFilePreview() - hideNotionPagePreview() - }} - > - - {t('datasetCreation.stepOne.dataSourceType.notion')} -
-
changeType(DataSourceType.WEB)} - > - - {t('datasetCreation.stepOne.dataSourceType.web')} -
+
+ { + shouldShowDataSourceTypeList && ( +
{t('datasetCreation.steps.one')}
+ ) + } +
+ { + shouldShowDataSourceTypeList && ( +
+
{ + if (dataSourceTypeDisable) + return + changeType(DataSourceType.FILE) + hideFilePreview() + hideNotionPagePreview() + }} + > + + {t('datasetCreation.stepOne.dataSourceType.file')}
- ) - } - {dataSourceType === DataSourceType.FILE && ( - <> - - {isShowVectorSpaceFull && ( -
- -
- )} -
- {/* */} - +
{ + if (dataSourceTypeDisable) + return + changeType(DataSourceType.NOTION) + hideFilePreview() + hideNotionPagePreview() + }} + > + + {t('datasetCreation.stepOne.dataSourceType.notion')}
- - )} - {dataSourceType === DataSourceType.NOTION && ( - <> - {!hasConnection && } - {hasConnection && ( - <> -
- page.page_id)} - onSelect={updateNotionPages} - onPreview={updateCurrentPage} - /> -
- {isShowVectorSpaceFull && ( -
- -
- )} -
- {/* */} - -
- - )} - - )} - {dataSourceType === DataSourceType.WEB && ( - <> -
- +
changeType(DataSourceType.WEB)} + > + + {t('datasetCreation.stepOne.dataSourceType.web')} +
+
+ ) + } + {dataSourceType === DataSourceType.FILE && ( + <> + + {isShowVectorSpaceFull && ( +
+
- {isShowVectorSpaceFull && ( -
- + )} + + + )} + {dataSourceType === DataSourceType.NOTION && ( + <> + {!hasConnection && } + {hasConnection && ( + <> +
+ page.page_id)} + onSelect={updateNotionPages} + onPreview={updateCurrentPage} + />
- )} -
- {/* */} - + {isShowVectorSpaceFull && ( +
+ +
+ )} + + + )} + + )} + {dataSourceType === DataSourceType.WEB && ( + <> +
+ +
+ {isShowVectorSpaceFull && ( +
+
- - )} - {!datasetId && ( - <> -
- - - {t('datasetCreation.stepOne.emptyDatasetCreation')} - - - )} -
- + )} + + + )} + {!datasetId && ( + <> +
+
{t('datasetCreation.stepOne.emptyDatasetCreation')}
+ + )}
+
-
- {currentFile && } - {currentNotionPage && } - {currentWebsite && } -
+ {currentFile && } + {currentNotionPage && } + {currentWebsite && }
) } diff --git a/web/app/components/datasets/create/step-three/index.tsx b/web/app/components/datasets/create/step-three/index.tsx index 8d979616d13b21..804a196ed5ac44 100644 --- a/web/app/components/datasets/create/step-three/index.tsx +++ b/web/app/components/datasets/create/step-three/index.tsx @@ -1,51 +1,45 @@ 'use client' import React from 'react' import { useTranslation } from 'react-i18next' -import { RiBookOpenLine } from '@remixicon/react' import EmbeddingProcess from '../embedding-process' +import s from './index.module.css' +import cn from '@/utils/classnames' import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' import type { FullDocumentDetail, createDocumentResponse } from '@/models/datasets' -import AppIcon from '@/app/components/base/app-icon' type StepThreeProps = { datasetId?: string datasetName?: string indexingType?: string - retrievalMethod?: string creationCache?: createDocumentResponse } -const StepThree = ({ datasetId, datasetName, indexingType, creationCache, retrievalMethod }: StepThreeProps) => { +const StepThree = ({ datasetId, datasetName, indexingType, creationCache }: StepThreeProps) => { const { t } = useTranslation() const media = useBreakpoints() const isMobile = media === MediaType.mobile return ( -
-
-
+
+
+
{!datasetId && ( <> -
-
{t('datasetCreation.stepThree.creationTitle')}
-
{t('datasetCreation.stepThree.creationContent')}
-
- -
-
{t('datasetCreation.stepThree.label')}
-
{datasetName || creationCache?.dataset?.name}
-
-
+
+
{t('datasetCreation.stepThree.creationTitle')}
+
{t('datasetCreation.stepThree.creationContent')}
+
{t('datasetCreation.stepThree.label')}
+
{datasetName || creationCache?.dataset?.name}
-
+
)} {datasetId && ( -
-
{t('datasetCreation.stepThree.additionTitle')}
-
{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}
+
+
{t('datasetCreation.stepThree.additionTitle')}
+
{`${t('datasetCreation.stepThree.additionP1')} ${datasetName || creationCache?.dataset?.name} ${t('datasetCreation.stepThree.additionP2')}`}
)}
- {!isMobile && ( -
-
-
- -
-
{t('datasetCreation.stepThree.sideTipTitle')}
-
{t('datasetCreation.stepThree.sideTipContent')}
-
+ {!isMobile &&
+
+ +
{t('datasetCreation.stepThree.sideTipTitle')}
+
{t('datasetCreation.stepThree.sideTipContent')}
- )} +
}
) } diff --git a/web/app/components/datasets/create/step-two/index.module.css b/web/app/components/datasets/create/step-two/index.module.css index 178cbeba857dad..f89d6d67ea7088 100644 --- a/web/app/components/datasets/create/step-two/index.module.css +++ b/web/app/components/datasets/create/step-two/index.module.css @@ -13,6 +13,18 @@ z-index: 10; } +.form { + @apply px-16 pb-8; +} + +.form .label { + @apply pt-6 pb-2 flex items-center; + font-weight: 500; + font-size: 16px; + line-height: 24px; + color: #344054; +} + .segmentationItem { min-height: 68px; } @@ -63,10 +75,6 @@ cursor: pointer; } -.disabled { - cursor: not-allowed !important; -} - .indexItem.disabled:hover { background-color: #fcfcfd; border-color: #f2f4f7; @@ -79,7 +87,8 @@ } .radioItem { - @apply relative mb-2 rounded-xl border border-components-option-card-option-border cursor-pointer bg-components-option-card-option-bg; + @apply relative mb-2 rounded-xl border border-gray-100 cursor-pointer; + background-color: #fcfcfd; } .radioItem.segmentationItem.custom { @@ -137,7 +146,7 @@ } .typeIcon.economical { - background-image: url(../assets/piggy-bank-mod.svg); + background-image: url(../assets/piggy-bank-01.svg); } .radioItem .radio { @@ -238,7 +247,7 @@ } .ruleItem { - @apply flex items-center py-1.5; + @apply flex items-center; } .formFooter { @@ -385,6 +394,19 @@ max-width: 524px; } +.previewHeader { + position: sticky; + top: 0; + left: 0; + padding-top: 42px; + background-color: #fff; + font-weight: 600; + font-size: 18px; + line-height: 28px; + color: #101828; + z-index: 10; +} + /* * `fixed` must under `previewHeader` because of style override would not work */ @@ -410,4 +432,4 @@ font-size: 12px; line-height: 18px; } -} +} \ No newline at end of file diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 0d7202967a5e9f..f915c68fef0345 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -1,80 +1,65 @@ 'use client' -import type { FC, PropsWithChildren } from 'react' -import React, { useCallback, useEffect, useRef, useState } from 'react' +import React, { useCallback, useEffect, useLayoutEffect, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' +import { useBoolean } from 'ahooks' +import { XMarkIcon } from '@heroicons/react/20/solid' +import { RocketLaunchIcon } from '@heroicons/react/24/outline' import { - RiAlertFill, - RiArrowLeftLine, - RiSearchEyeLine, + RiCloseLine, } from '@remixicon/react' import Link from 'next/link' -import Image from 'next/image' -import { useHover } from 'ahooks' -import SettingCog from '../assets/setting-gear-mod.svg' -import OrangeEffect from '../assets/option-card-effect-orange.svg' -import FamilyMod from '../assets/family-mod.svg' -import Note from '../assets/note-mod.svg' -import FileList from '../assets/file-list-3-fill.svg' -import { indexMethodIcon } from '../icons' -import { PreviewContainer } from '../../preview/container' -import { ChunkContainer, QAPreview } from '../../chunk' -import { PreviewHeader } from '../../preview/header' -import { FormattedText } from '../../formatted-text/formatted' -import { PreviewSlice } from '../../formatted-text/flavours/preview-slice' -import PreviewDocumentPicker from '../../common/document-picker/preview-document-picker' +import { groupBy } from 'lodash-es' +import PreviewItem, { PreviewType } from './preview-item' +import LanguageSelect from './language-select' import s from './index.module.css' import unescape from './unescape' import escape from './escape' -import { OptionCard } from './option-card' -import LanguageSelect from './language-select' -import { DelimiterInput, MaxLengthInput, OverlapInput } from './inputs' import cn from '@/utils/classnames' -import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, DocumentItem, FullDocumentDetail, ParentMode, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' - +import type { CrawlOptions, CrawlResultItem, CreateDocumentReq, CustomFile, FileIndexingEstimateResponse, FullDocumentDetail, IndexingEstimateParams, NotionInfo, PreProcessingRule, ProcessRule, Rules, createDocumentResponse } from '@/models/datasets' +import { + createDocument, + createFirstDocument, + fetchFileIndexingEstimate as didFetchFileIndexingEstimate, + fetchDefaultProcessRule, +} from '@/service/datasets' import Button from '@/app/components/base/button' +import Input from '@/app/components/base/input' +import Loading from '@/app/components/base/loading' import FloatRightContainer from '@/app/components/base/float-right-container' import RetrievalMethodConfig from '@/app/components/datasets/common/retrieval-method-config' import EconomicalRetrievalMethodConfig from '@/app/components/datasets/common/economical-retrieval-method-config' import { type RetrievalConfig } from '@/types/app' import { ensureRerankModelSelected, isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model' import Toast from '@/app/components/base/toast' +import { formatNumber } from '@/utils/format' import type { NotionPage } from '@/models/common' import { DataSourceProvider } from '@/models/common' -import { ChunkingMode, DataSourceType, RerankingModeEnum } from '@/models/datasets' +import { DataSourceType, DocForm } from '@/models/datasets' +import NotionIcon from '@/app/components/base/notion-icon' +import Switch from '@/app/components/base/switch' +import { MessageChatSquare } from '@/app/components/base/icons/src/public/common' import { useDatasetDetailContext } from '@/context/dataset-detail' import I18n from '@/context/i18n' +import { IS_CE_EDITION } from '@/config' import { RETRIEVE_METHOD } from '@/types/app' import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints' +import Tooltip from '@/app/components/base/tooltip' import { useDefaultModel, useModelList, useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks' import { LanguagesSupported } from '@/i18n/language' import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector' import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations' import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations' -import Checkbox from '@/app/components/base/checkbox' -import RadioCard from '@/app/components/base/radio-card' -import { IS_CE_EDITION } from '@/config' -import Divider from '@/app/components/base/divider' -import { getNotionInfo, getWebsiteInfo, useCreateDocument, useCreateFirstDocument, useFetchDefaultProcessRule, useFetchFileIndexingEstimateForFile, useFetchFileIndexingEstimateForNotion, useFetchFileIndexingEstimateForWeb } from '@/service/knowledge/use-create-dataset' -import Badge from '@/app/components/base/badge' -import { SkeletonContainer, SkeletonPoint, SkeletonRectangle, SkeletonRow } from '@/app/components/base/skeleton' -import Tooltip from '@/app/components/base/tooltip' -import CustomDialog from '@/app/components/base/dialog' -import { PortalToFollowElem, PortalToFollowElemContent, PortalToFollowElemTrigger } from '@/app/components/base/portal-to-follow-elem' -import { AlertTriangle } from '@/app/components/base/icons/src/vender/solid/alertsAndFeedback' - -const TextLabel: FC = (props) => { - return -} +import { Globe01 } from '@/app/components/base/icons/src/vender/line/mapsAndTravel' +type ValueOf = T[keyof T] type StepTwoProps = { isSetting?: boolean documentDetail?: FullDocumentDetail isAPIKeySet: boolean onSetting: () => void datasetId?: string - indexingType?: IndexingType - retrievalMethod?: string + indexingType?: ValueOf dataSourceType: DataSourceType files: CustomFile[] notionPages?: NotionPage[] @@ -84,48 +69,21 @@ type StepTwoProps = { websiteCrawlJobId?: string onStepChange?: (delta: number) => void updateIndexingTypeCache?: (type: string) => void - updateRetrievalMethodCache?: (method: string) => void updateResultCache?: (res: createDocumentResponse) => void onSave?: () => void onCancel?: () => void } -export enum SegmentType { +enum SegmentType { AUTO = 'automatic', CUSTOM = 'custom', } -export enum IndexingType { +enum IndexingType { QUALIFIED = 'high_quality', ECONOMICAL = 'economy', } const DEFAULT_SEGMENT_IDENTIFIER = '\\n\\n' -const DEFAULT_MAXMIMUM_CHUNK_LENGTH = 500 -const DEFAULT_OVERLAP = 50 - -type ParentChildConfig = { - chunkForContext: ParentMode - parent: { - delimiter: string - maxLength: number - } - child: { - delimiter: string - maxLength: number - } -} - -const defaultParentChildConfig: ParentChildConfig = { - chunkForContext: 'paragraph', - parent: { - delimiter: '\\n\\n', - maxLength: 500, - }, - child: { - delimiter: '\\n', - maxLength: 200, - }, -} const StepTwo = ({ isSetting, @@ -146,7 +104,6 @@ const StepTwo = ({ updateResultCache, onSave, onCancel, - updateRetrievalMethodCache, }: StepTwoProps) => { const { t } = useTranslation() const { locale } = useContext(I18n) @@ -154,166 +111,66 @@ const StepTwo = ({ const isMobile = media === MediaType.mobile const { dataset: currentDataset, mutateDatasetRes } = useDatasetDetailContext() - - const isInUpload = Boolean(currentDataset) - const isUploadInEmptyDataset = isInUpload && !currentDataset?.doc_form - const isNotUploadInEmptyDataset = !isUploadInEmptyDataset - const isInInit = !isInUpload && !isSetting - const isInCreatePage = !datasetId || (datasetId && !currentDataset?.data_source_type) const dataSourceType = isInCreatePage ? inCreatePageDataSourceType : currentDataset?.data_source_type - const [segmentationType, setSegmentationType] = useState(SegmentType.CUSTOM) + const scrollRef = useRef(null) + const [scrolled, setScrolled] = useState(false) + const previewScrollRef = useRef(null) + const [previewScrolled, setPreviewScrolled] = useState(false) + const [segmentationType, setSegmentationType] = useState(SegmentType.AUTO) const [segmentIdentifier, doSetSegmentIdentifier] = useState(DEFAULT_SEGMENT_IDENTIFIER) - const setSegmentIdentifier = useCallback((value: string, canEmpty?: boolean) => { - doSetSegmentIdentifier(value ? escape(value) : (canEmpty ? '' : DEFAULT_SEGMENT_IDENTIFIER)) + const setSegmentIdentifier = useCallback((value: string) => { + doSetSegmentIdentifier(value ? escape(value) : DEFAULT_SEGMENT_IDENTIFIER) }, []) - const [maxChunkLength, setMaxChunkLength] = useState(DEFAULT_MAXMIMUM_CHUNK_LENGTH) // default chunk length + const [maxChunkLength, setMaxChunkLength] = useState(4000) // default chunk length const [limitMaxChunkLength, setLimitMaxChunkLength] = useState(4000) - const [overlap, setOverlap] = useState(DEFAULT_OVERLAP) + const [overlap, setOverlap] = useState(50) const [rules, setRules] = useState([]) const [defaultConfig, setDefaultConfig] = useState() const hasSetIndexType = !!indexingType - const [indexType, setIndexType] = useState( + const [indexType, setIndexType] = useState>( (indexingType || isAPIKeySet) ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL, ) - - const [previewFile, setPreviewFile] = useState( - (datasetId && documentDetail) - ? documentDetail.file - : files[0], - ) - const [previewNotionPage, setPreviewNotionPage] = useState( - (datasetId && documentDetail) - ? documentDetail.notion_page - : notionPages[0], - ) - - const [previewWebsitePage, setPreviewWebsitePage] = useState( - (datasetId && documentDetail) - ? documentDetail.website_page - : websitePages[0], - ) - - // QA Related - const [isLanguageSelectDisabled, _setIsLanguageSelectDisabled] = useState(false) - const [isQAConfirmDialogOpen, setIsQAConfirmDialogOpen] = useState(false) - const [docForm, setDocForm] = useState( - (datasetId && documentDetail) ? documentDetail.doc_form as ChunkingMode : ChunkingMode.text, + const [isLanguageSelectDisabled, setIsLanguageSelectDisabled] = useState(false) + const [docForm, setDocForm] = useState( + (datasetId && documentDetail) ? documentDetail.doc_form : DocForm.TEXT, ) - const handleChangeDocform = (value: ChunkingMode) => { - if (value === ChunkingMode.qa && indexType === IndexingType.ECONOMICAL) { - setIsQAConfirmDialogOpen(true) - return - } - if (value === ChunkingMode.parentChild && indexType === IndexingType.ECONOMICAL) - setIndexType(IndexingType.QUALIFIED) - setDocForm(value) - // eslint-disable-next-line @typescript-eslint/no-use-before-define - currentEstimateMutation.reset() - } - const [docLanguage, setDocLanguage] = useState( (datasetId && documentDetail) ? documentDetail.doc_language : (locale !== LanguagesSupported[1] ? 'English' : 'Chinese'), ) + const [QATipHide, setQATipHide] = useState(false) + const [previewSwitched, setPreviewSwitched] = useState(false) + const [showPreview, { setTrue: setShowPreview, setFalse: hidePreview }] = useBoolean() + const [customFileIndexingEstimate, setCustomFileIndexingEstimate] = useState(null) + const [automaticFileIndexingEstimate, setAutomaticFileIndexingEstimate] = useState(null) - const [parentChildConfig, setParentChildConfig] = useState(defaultParentChildConfig) + const fileIndexingEstimate = (() => { + return segmentationType === SegmentType.AUTO ? automaticFileIndexingEstimate : customFileIndexingEstimate + })() + const [isCreating, setIsCreating] = useState(false) - const getIndexing_technique = () => indexingType || indexType - const currentDocForm = currentDataset?.doc_form || docForm + const scrollHandle = (e: Event) => { + if ((e.target as HTMLDivElement).scrollTop > 0) + setScrolled(true) - const getProcessRule = (): ProcessRule => { - if (currentDocForm === ChunkingMode.parentChild) { - return { - rules: { - pre_processing_rules: rules, - segmentation: { - separator: unescape( - parentChildConfig.parent.delimiter, - ), - max_tokens: parentChildConfig.parent.maxLength, - }, - parent_mode: parentChildConfig.chunkForContext, - subchunk_segmentation: { - separator: unescape(parentChildConfig.child.delimiter), - max_tokens: parentChildConfig.child.maxLength, - }, - }, - mode: 'hierarchical', - } as ProcessRule - } - return { - rules: { - pre_processing_rules: rules, - segmentation: { - separator: unescape(segmentIdentifier), - max_tokens: maxChunkLength, - chunk_overlap: overlap, - }, - }, // api will check this. It will be removed after api refactored. - mode: segmentationType, - } as ProcessRule + else + setScrolled(false) } - const fileIndexingEstimateQuery = useFetchFileIndexingEstimateForFile({ - docForm: currentDocForm, - docLanguage, - dataSourceType: DataSourceType.FILE, - files: previewFile - ? [files.find(file => file.name === previewFile.name)!] - : files, - indexingTechnique: getIndexing_technique() as any, - processRule: getProcessRule(), - dataset_id: datasetId!, - }) - const notionIndexingEstimateQuery = useFetchFileIndexingEstimateForNotion({ - docForm: currentDocForm, - docLanguage, - dataSourceType: DataSourceType.NOTION, - notionPages: [previewNotionPage], - indexingTechnique: getIndexing_technique() as any, - processRule: getProcessRule(), - dataset_id: datasetId || '', - }) - - const websiteIndexingEstimateQuery = useFetchFileIndexingEstimateForWeb({ - docForm: currentDocForm, - docLanguage, - dataSourceType: DataSourceType.WEB, - websitePages: [previewWebsitePage], - crawlOptions, - websiteCrawlProvider, - websiteCrawlJobId, - indexingTechnique: getIndexing_technique() as any, - processRule: getProcessRule(), - dataset_id: datasetId || '', - }) - - const currentEstimateMutation = dataSourceType === DataSourceType.FILE - ? fileIndexingEstimateQuery - : dataSourceType === DataSourceType.NOTION - ? notionIndexingEstimateQuery - : websiteIndexingEstimateQuery - - const fetchEstimate = useCallback(() => { - if (dataSourceType === DataSourceType.FILE) - fileIndexingEstimateQuery.mutate() - - if (dataSourceType === DataSourceType.NOTION) - notionIndexingEstimateQuery.mutate() + const previewScrollHandle = (e: Event) => { + if ((e.target as HTMLDivElement).scrollTop > 0) + setPreviewScrolled(true) - if (dataSourceType === DataSourceType.WEB) - websiteIndexingEstimateQuery.mutate() - }, [dataSourceType, fileIndexingEstimateQuery, notionIndexingEstimateQuery, websiteIndexingEstimateQuery]) - - const estimate - = dataSourceType === DataSourceType.FILE - ? fileIndexingEstimateQuery.data - : dataSourceType === DataSourceType.NOTION - ? notionIndexingEstimateQuery.data - : websiteIndexingEstimateQuery.data + else + setPreviewScrolled(false) + } + const getFileName = (name: string) => { + const arr = name.split('.') + return arr.slice(0, -1).join('.') + } const getRuleName = (key: string) => { if (key === 'remove_extra_spaces') @@ -341,20 +198,128 @@ const StepTwo = ({ if (defaultConfig) { setSegmentIdentifier(defaultConfig.segmentation.separator) setMaxChunkLength(defaultConfig.segmentation.max_tokens) - setOverlap(defaultConfig.segmentation.chunk_overlap!) + setOverlap(defaultConfig.segmentation.chunk_overlap) setRules(defaultConfig.pre_processing_rules) } - setParentChildConfig(defaultParentChildConfig) } - const updatePreview = () => { - if (segmentationType === SegmentType.CUSTOM && maxChunkLength > 4000) { - Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck') }) + const fetchFileIndexingEstimate = async (docForm = DocForm.TEXT, language?: string) => { + // eslint-disable-next-line @typescript-eslint/no-use-before-define + const res = await didFetchFileIndexingEstimate(getFileIndexingEstimateParams(docForm, language)!) + if (segmentationType === SegmentType.CUSTOM) + setCustomFileIndexingEstimate(res) + else + setAutomaticFileIndexingEstimate(res) + } + + const confirmChangeCustomConfig = () => { + if (segmentationType === SegmentType.CUSTOM && maxChunkLength > limitMaxChunkLength) { + Toast.notify({ type: 'error', message: t('datasetCreation.stepTwo.maxLengthCheck', { limit: limitMaxChunkLength }) }) return } - fetchEstimate() + setCustomFileIndexingEstimate(null) + setShowPreview() + fetchFileIndexingEstimate() + setPreviewSwitched(false) + } + + const getIndexing_technique = () => indexingType || indexType + + const getProcessRule = () => { + const processRule: ProcessRule = { + rules: {} as any, // api will check this. It will be removed after api refactored. + mode: segmentationType, + } + if (segmentationType === SegmentType.CUSTOM) { + const ruleObj = { + pre_processing_rules: rules, + segmentation: { + separator: unescape(segmentIdentifier), + max_tokens: maxChunkLength, + chunk_overlap: overlap, + }, + } + processRule.rules = ruleObj + } + return processRule + } + + const getNotionInfo = () => { + const workspacesMap = groupBy(notionPages, 'workspace_id') + const workspaces = Object.keys(workspacesMap).map((workspaceId) => { + return { + workspaceId, + pages: workspacesMap[workspaceId], + } + }) + return workspaces.map((workspace) => { + return { + workspace_id: workspace.workspaceId, + pages: workspace.pages.map((page) => { + const { page_id, page_name, page_icon, type } = page + return { + page_id, + page_name, + page_icon, + type, + } + }), + } + }) as NotionInfo[] + } + + const getWebsiteInfo = () => { + return { + provider: websiteCrawlProvider, + job_id: websiteCrawlJobId, + urls: websitePages.map(page => page.source_url), + only_main_content: crawlOptions?.only_main_content, + } } + const getFileIndexingEstimateParams = (docForm: DocForm, language?: string): IndexingEstimateParams | undefined => { + if (dataSourceType === DataSourceType.FILE) { + return { + info_list: { + data_source_type: dataSourceType, + file_info_list: { + file_ids: files.map(file => file.id) as string[], + }, + }, + indexing_technique: getIndexing_technique() as string, + process_rule: getProcessRule(), + doc_form: docForm, + doc_language: language || docLanguage, + dataset_id: datasetId as string, + } + } + if (dataSourceType === DataSourceType.NOTION) { + return { + info_list: { + data_source_type: dataSourceType, + notion_info_list: getNotionInfo(), + }, + indexing_technique: getIndexing_technique() as string, + process_rule: getProcessRule(), + doc_form: docForm, + doc_language: language || docLanguage, + dataset_id: datasetId as string, + } + } + if (dataSourceType === DataSourceType.WEB) { + return { + info_list: { + data_source_type: dataSourceType, + website_info_list: getWebsiteInfo(), + }, + indexing_technique: getIndexing_technique() as string, + process_rule: getProcessRule(), + doc_form: docForm, + doc_language: language || docLanguage, + dataset_id: datasetId as string, + } + } + } const { modelList: rerankModelList, defaultModel: rerankDefaultModel, @@ -386,14 +351,13 @@ const StepTwo = ({ if (isSetting) { params = { original_document_id: documentDetail?.id, - doc_form: currentDocForm, + doc_form: docForm, doc_language: docLanguage, process_rule: getProcessRule(), // eslint-disable-next-line @typescript-eslint/no-use-before-define retrieval_model: retrievalConfig, // Readonly. If want to changed, just go to settings page. embedding_model: embeddingModel.model, // Readonly embedding_model_provider: embeddingModel.provider, // Readonly - indexing_technique: getIndexing_technique(), } as CreateDocumentReq } else { // create @@ -413,12 +377,8 @@ const StepTwo = ({ } const postRetrievalConfig = ensureRerankModelSelected({ rerankDefaultModel: rerankDefaultModel!, - retrievalConfig: { - // eslint-disable-next-line @typescript-eslint/no-use-before-define - ...retrievalConfig, - // eslint-disable-next-line @typescript-eslint/no-use-before-define - reranking_enable: retrievalConfig.reranking_mode === RerankingModeEnum.RerankingModel, - }, + // eslint-disable-next-line @typescript-eslint/no-use-before-define + retrievalConfig, indexMethod: indexMethod as string, }) params = { @@ -430,7 +390,7 @@ const StepTwo = ({ }, indexing_technique: getIndexing_technique(), process_rule: getProcessRule(), - doc_form: currentDocForm, + doc_form: docForm, doc_language: docLanguage, retrieval_model: postRetrievalConfig, @@ -443,36 +403,29 @@ const StepTwo = ({ } } if (dataSourceType === DataSourceType.NOTION) - params.data_source.info_list.notion_info_list = getNotionInfo(notionPages) + params.data_source.info_list.notion_info_list = getNotionInfo() - if (dataSourceType === DataSourceType.WEB) { - params.data_source.info_list.website_info_list = getWebsiteInfo({ - websiteCrawlProvider, - websiteCrawlJobId, - websitePages, - }) - } + if (dataSourceType === DataSourceType.WEB) + params.data_source.info_list.website_info_list = getWebsiteInfo() } return params } - const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule({ - onSuccess(data) { - const separator = data.rules.segmentation.separator + const getRules = async () => { + try { + const res = await fetchDefaultProcessRule({ url: '/datasets/process-rule' }) + const separator = res.rules.segmentation.separator setSegmentIdentifier(separator) - setMaxChunkLength(data.rules.segmentation.max_tokens) - setOverlap(data.rules.segmentation.chunk_overlap!) - setRules(data.rules.pre_processing_rules) - setDefaultConfig(data.rules) - setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length) - }, - onError(error) { - Toast.notify({ - type: 'error', - message: `${error}`, - }) - }, - }) + setMaxChunkLength(res.rules.segmentation.max_tokens) + setLimitMaxChunkLength(res.limits.indexing_max_segmentation_tokens_length) + setOverlap(res.rules.segmentation.chunk_overlap) + setRules(res.rules.pre_processing_rules) + setDefaultConfig(res.rules) + } + catch (err) { + console.log(err) + } + } const getRulesFromDetail = () => { if (documentDetail) { @@ -482,7 +435,7 @@ const StepTwo = ({ const overlap = rules.segmentation.chunk_overlap setSegmentIdentifier(separator) setMaxChunkLength(max) - setOverlap(overlap!) + setOverlap(overlap) setRules(rules.pre_processing_rules) setDefaultConfig(rules) } @@ -490,81 +443,119 @@ const StepTwo = ({ const getDefaultMode = () => { if (documentDetail) - // @ts-expect-error fix after api refactored setSegmentationType(documentDetail.dataset_process_rule.mode) } - const createFirstDocumentMutation = useCreateFirstDocument({ - onError(error) { - Toast.notify({ - type: 'error', - message: `${error}`, - }) - }, - }) - const createDocumentMutation = useCreateDocument(datasetId!, { - onError(error) { + const createHandle = async () => { + if (isCreating) + return + setIsCreating(true) + try { + let res + const params = getCreationParams() + if (!params) + return false + + setIsCreating(true) + if (!datasetId) { + res = await createFirstDocument({ + body: params as CreateDocumentReq, + }) + updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) + updateResultCache && updateResultCache(res) + } + else { + res = await createDocument({ + datasetId, + body: params as CreateDocumentReq, + }) + updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) + updateResultCache && updateResultCache(res) + } + if (mutateDatasetRes) + mutateDatasetRes() + onStepChange && onStepChange(+1) + isSetting && onSave && onSave() + } + catch (err) { Toast.notify({ type: 'error', - message: `${error}`, + message: `${err}`, }) - }, - }) - - const isCreating = createFirstDocumentMutation.isPending || createDocumentMutation.isPending + } + finally { + setIsCreating(false) + } + } - const createHandle = async () => { - const params = getCreationParams() - if (!params) - return false + const handleSwitch = (state: boolean) => { + if (state) + setDocForm(DocForm.QA) + else + setDocForm(DocForm.TEXT) + } - if (!datasetId) { - await createFirstDocumentMutation.mutateAsync( - params, - { - onSuccess(data) { - updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) - updateResultCache && updateResultCache(data) - // eslint-disable-next-line @typescript-eslint/no-use-before-define - updateRetrievalMethodCache && updateRetrievalMethodCache(retrievalConfig.search_method as string) - }, - }, - ) + const previewSwitch = async (language?: string) => { + setPreviewSwitched(true) + setIsLanguageSelectDisabled(true) + if (segmentationType === SegmentType.AUTO) + setAutomaticFileIndexingEstimate(null) + else + setCustomFileIndexingEstimate(null) + try { + await fetchFileIndexingEstimate(DocForm.QA, language) } - else { - await createDocumentMutation.mutateAsync(params, { - onSuccess(data) { - updateIndexingTypeCache && updateIndexingTypeCache(indexType as string) - updateResultCache && updateResultCache(data) - }, - }) + finally { + setIsLanguageSelectDisabled(false) } - if (mutateDatasetRes) - mutateDatasetRes() - onStepChange && onStepChange(+1) - isSetting && onSave && onSave() } - const changeToEconomicalType = () => { - if (docForm !== ChunkingMode.text) - return + const handleSelect = (language: string) => { + setDocLanguage(language) + // Switch language, re-cutter + if (docForm === DocForm.QA && previewSwitched) + previewSwitch(language) + } - if (!hasSetIndexType) + const changeToEconomicalType = () => { + if (!hasSetIndexType) { setIndexType(IndexingType.ECONOMICAL) + setDocForm(DocForm.TEXT) + } } useEffect(() => { // fetch rules if (!isSetting) { - fetchDefaultProcessRuleMutation.mutate('/datasets/process-rule') + getRules() } else { getRulesFromDetail() getDefaultMode() } - // eslint-disable-next-line react-hooks/exhaustive-deps }, []) + useEffect(() => { + scrollRef.current?.addEventListener('scroll', scrollHandle) + return () => { + scrollRef.current?.removeEventListener('scroll', scrollHandle) + } + }, []) + + useLayoutEffect(() => { + if (showPreview) { + previewScrollRef.current?.addEventListener('scroll', previewScrollHandle) + return () => { + previewScrollRef.current?.removeEventListener('scroll', previewScrollHandle) + } + } + }, [showPreview]) + + useEffect(() => { + if (indexingType === IndexingType.ECONOMICAL && docForm === DocForm.QA) + setDocForm(DocForm.TEXT) + }, [indexingType, docForm]) + useEffect(() => { // get indexing type by props if (indexingType) @@ -574,6 +565,20 @@ const StepTwo = ({ setIndexType(isAPIKeySet ? IndexingType.QUALIFIED : IndexingType.ECONOMICAL) }, [isAPIKeySet, indexingType, datasetId]) + useEffect(() => { + if (segmentationType === SegmentType.AUTO) { + setAutomaticFileIndexingEstimate(null) + !isMobile && setShowPreview() + fetchFileIndexingEstimate() + setPreviewSwitched(false) + } + else { + hidePreview() + setCustomFileIndexingEstimate(null) + setPreviewSwitched(false) + } + }, [segmentationType, indexType]) + const [retrievalConfig, setRetrievalConfig] = useState(currentDataset?.retrieval_model_dict || { search_method: RETRIEVE_METHOD.semantic, reranking_enable: false, @@ -586,589 +591,433 @@ const StepTwo = ({ score_threshold: 0.5, } as RetrievalConfig) - const economyDomRef = useRef(null) - const isHoveringEconomy = useHover(economyDomRef) - return (
-
-
{t('datasetCreation.stepTwo.segmentation')}
- {((isInUpload && [ChunkingMode.text, ChunkingMode.qa].includes(currentDataset!.doc_form)) - || isUploadInEmptyDataset - || isInInit) - && } - activeHeaderClassName='bg-dataset-option-card-blue-gradient' - description={t('datasetCreation.stepTwo.generalTip')} - isActive={ - [ChunkingMode.text, ChunkingMode.qa].includes(currentDocForm) - } - onSwitched={() => - handleChangeDocform(ChunkingMode.text) - } - actions={ - <> - - - - } - noHighlight={isInUpload && isNotUploadInEmptyDataset} - > -
-
- setSegmentIdentifier(e.target.value, true)} - /> - - +
+
+ {t('datasetCreation.steps.two')} + {(isMobile || !showPreview) && ( + + )} +
+
+
{t('datasetCreation.stepTwo.segmentation')}
+
+
setSegmentationType(SegmentType.AUTO)} + > + + +
+
{t('datasetCreation.stepTwo.auto')}
+
{t('datasetCreation.stepTwo.autoDescription')}
-
-
-
- {t('datasetCreation.stepTwo.rules')} +
+
setSegmentationType(SegmentType.CUSTOM)} + > + + +
+
{t('datasetCreation.stepTwo.custom')}
+
{t('datasetCreation.stepTwo.customDescription')}
+
+ {segmentationType === SegmentType.CUSTOM && ( +
+
+
+
+ {t('datasetCreation.stepTwo.separator')} + + {t('datasetCreation.stepTwo.separatorTip')} +
+ } + /> +
+ setSegmentIdentifier(e.target.value)} + /> +
- -
-
- {rules.map(rule => ( -
{ - ruleChangeHandle(rule.id) - }}> - +
+
{t('datasetCreation.stepTwo.maxLength')}
+ setMaxChunkLength(parseInt(e.target.value.replace(/^0+/, ''), 10))} /> -
- ))} - {IS_CE_EDITION && <> - -
-
{ - if (currentDataset?.doc_form) - return - if (docForm === ChunkingMode.qa) - handleChangeDocform(ChunkingMode.text) - else - handleChangeDocform(ChunkingMode.qa) - }}> - +
+
+
+ {t('datasetCreation.stepTwo.overlap')} + + {t('datasetCreation.stepTwo.overlapTip')} +
+ } /> -
- setOverlap(parseInt(e.target.value.replace(/^0+/, ''), 10))} /> -
- {currentDocForm === ChunkingMode.qa && ( -
- - - {t('datasetCreation.stepTwo.QATip')} - -
- )} - } -
-
-
- } - { - ( - (isInUpload && currentDataset!.doc_form === ChunkingMode.parentChild) - || isUploadInEmptyDataset - || isInInit - ) - && } - effectImg={OrangeEffect.src} - activeHeaderClassName='bg-dataset-option-card-orange-gradient' - description={t('datasetCreation.stepTwo.parentChildTip')} - isActive={currentDocForm === ChunkingMode.parentChild} - onSwitched={() => handleChangeDocform(ChunkingMode.parentChild)} - actions={ - <> - - - - } - noHighlight={isInUpload && isNotUploadInEmptyDataset} - > -
-
-
-
- {t('datasetCreation.stepTwo.parentChunkForContext')}
- -
- } - title={t('datasetCreation.stepTwo.paragraph')} - description={t('datasetCreation.stepTwo.paragraphTip')} - isChosen={parentChildConfig.chunkForContext === 'paragraph'} - onChosen={() => setParentChildConfig( - { - ...parentChildConfig, - chunkForContext: 'paragraph', - }, - )} - chosenConfig={ -
- setParentChildConfig({ - ...parentChildConfig, - parent: { - ...parentChildConfig.parent, - delimiter: e.target.value ? escape(e.target.value) : '', - }, - })} - /> - setParentChildConfig({ - ...parentChildConfig, - parent: { - ...parentChildConfig.parent, - maxLength: value, - }, - })} - /> +
+
+
{t('datasetCreation.stepTwo.rules')}
+ {rules.map(rule => ( +
+ ruleChangeHandle(rule.id)} className="w-4 h-4 rounded border-gray-300 text-blue-700 focus:ring-blue-700" /> + +
+ ))}
- } - /> - } - title={t('datasetCreation.stepTwo.fullDoc')} - description={t('datasetCreation.stepTwo.fullDocTip')} - onChosen={() => setParentChildConfig( - { - ...parentChildConfig, - chunkForContext: 'full-doc', - }, - )} - isChosen={parentChildConfig.chunkForContext === 'full-doc'} - /> -
- -
-
-
- {t('datasetCreation.stepTwo.childChunkForRetrieval')}
- -
-
- setParentChildConfig({ - ...parentChildConfig, - child: { - ...parentChildConfig.child, - delimiter: e.target.value ? escape(e.target.value) : '', - }, - })} - /> - setParentChildConfig({ - ...parentChildConfig, - child: { - ...parentChildConfig.child, - maxLength: value, - }, - })} - /> -
-
-
-
-
- {t('datasetCreation.stepTwo.rules')} +
+ +
-
-
- {rules.map(rule => ( -
{ - ruleChangeHandle(rule.id) - }}> - - -
- ))} -
-
+ )}
- } - -
{t('datasetCreation.stepTwo.indexMode')}
-
- {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && ( - - {t('datasetCreation.stepTwo.qualified')} - - {t('datasetCreation.stepTwo.recommend')} - - +
+
{t('datasetCreation.stepTwo.indexMode')}
+
+
+ {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.QUALIFIED)) && ( +
{ + if (isAPIKeySet) + setIndexType(IndexingType.QUALIFIED) + }} + > + {!hasSetIndexType && } - -
} - description={t('datasetCreation.stepTwo.qualifiedTip')} - icon={} - isActive={!hasSetIndexType && indexType === IndexingType.QUALIFIED} - disabled={!isAPIKeySet || hasSetIndexType} - onSwitched={() => { - if (isAPIKeySet) - setIndexType(IndexingType.QUALIFIED) - }} - /> - )} - - {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && ( - <> - setIsQAConfirmDialogOpen(false)} className='w-[432px]'> -
-

- {t('datasetCreation.stepTwo.qaSwitchHighQualityTipTitle')} -

-

- {t('datasetCreation.stepTwo.qaSwitchHighQualityTipContent')} -

-
-
- - +
+
+ {t('datasetCreation.stepTwo.qualified')} + {!hasSetIndexType && {t('datasetCreation.stepTwo.recommend')}} +
+
{t('datasetCreation.stepTwo.qualifiedTip')}
+
+ {!isAPIKeySet && ( +
+ {t('datasetCreation.stepTwo.warning')}  + {t('datasetCreation.stepTwo.click')} +
+ )}
- - - - } - isActive={!hasSetIndexType && indexType === IndexingType.ECONOMICAL} - disabled={!isAPIKeySet || hasSetIndexType || docForm !== ChunkingMode.text} - ref={economyDomRef} - onSwitched={() => { - if (isAPIKeySet && docForm === ChunkingMode.text) - setIndexType(IndexingType.ECONOMICAL) - }} - /> - - -
- { - docForm === ChunkingMode.qa - ? t('datasetCreation.stepTwo.notAvailableForQA') - : t('datasetCreation.stepTwo.notAvailableForParentChild') - } + )} + + {(!hasSetIndexType || (hasSetIndexType && indexingType === IndexingType.ECONOMICAL)) && ( +
+ + {!hasSetIndexType && } +
+
{t('datasetCreation.stepTwo.economical')}
+
{t('datasetCreation.stepTwo.economicalTip')}
- - - )} -
- {!hasSetIndexType && indexType === IndexingType.QUALIFIED && ( -
-
-
- +
+ )}
- {t('datasetCreation.stepTwo.highQualityTip')} -
- )} - {hasSetIndexType && indexType === IndexingType.ECONOMICAL && ( -
- {t('datasetCreation.stepTwo.indexSettingTip')} - {t('datasetCreation.stepTwo.datasetSettingLink')} -
- )} - {/* Embedding model */} - {indexType === IndexingType.QUALIFIED && ( -
-
{t('datasetSettings.form.embeddingModel')}
- { - setEmbeddingModel(model) - }} - /> - {!!datasetId && ( -
+ {hasSetIndexType && indexType === IndexingType.ECONOMICAL && ( +
{t('datasetCreation.stepTwo.indexSettingTip')} - {t('datasetCreation.stepTwo.datasetSettingLink')} + {t('datasetCreation.stepTwo.datasetSettingLink')}
)} -
- )} - - {/* Retrieval Method Config */} -
- {!datasetId - ? ( -
-
{t('datasetSettings.form.retrievalSetting.title')}
-
- {t('datasetSettings.form.retrievalSetting.learnMore')} - {t('datasetSettings.form.retrievalSetting.longDescription')} + {IS_CE_EDITION && indexType === IndexingType.QUALIFIED && ( +
+
+
+ +
+
+
{t('datasetCreation.stepTwo.QATitle')}
+
+ {t('datasetCreation.stepTwo.QALanguage')} + +
+
+
+ +
+ {docForm === DocForm.QA && !QATipHide && ( +
+ {t('datasetCreation.stepTwo.QATip')} + setQATipHide(true)} /> +
+ )}
- ) - : ( -
-
{t('datasetSettings.form.retrievalSetting.title')}
+ )} + {/* Embedding model */} + {indexType === IndexingType.QUALIFIED && ( +
+
{t('datasetSettings.form.embeddingModel')}
+ { + setEmbeddingModel(model) + }} + /> + {!!datasetId && ( +
+ {t('datasetCreation.stepTwo.indexSettingTip')} + {t('datasetCreation.stepTwo.datasetSettingLink')} +
+ )}
)} - -
- { - getIndexing_technique() === IndexingType.QUALIFIED + {/* Retrieval Method Config */} +
+ {!datasetId ? ( - +
+
{t('datasetSettings.form.retrievalSetting.title')}
+
+ {t('datasetSettings.form.retrievalSetting.learnMore')} + {t('datasetSettings.form.retrievalSetting.longDescription')} +
+
) : ( - - ) - } -
-
+
+
{t('datasetSettings.form.retrievalSetting.title')}
+
+ )} - {!isSetting - ? ( -
- - -
- ) - : ( -
- - +
+ { + getIndexing_technique() === IndexingType.QUALIFIED + ? ( + + ) + : ( + + ) + } +
- )} -
- { }} footer={null}> - -
- {dataSourceType === DataSourceType.FILE - && >} - onChange={(selected) => { - currentEstimateMutation.reset() - setPreviewFile(selected) - currentEstimateMutation.mutate() - }} - // when it is from setting, it just has one file - value={isSetting ? (files[0]! as Required) : previewFile} - /> - } - {dataSourceType === DataSourceType.NOTION - && ({ - id: page.page_id, - name: page.page_name, - extension: 'md', - })) - } - onChange={(selected) => { - currentEstimateMutation.reset() - const selectedPage = notionPages.find(page => page.page_id === selected.id) - setPreviewNotionPage(selectedPage!) - currentEstimateMutation.mutate() - }} - value={{ - id: previewNotionPage?.page_id || '', - name: previewNotionPage?.page_name || '', - extension: 'md', - }} - /> - } - {dataSourceType === DataSourceType.WEB - && ({ - id: page.source_url, - name: page.title, - extension: 'md', - })) - } - onChange={(selected) => { - currentEstimateMutation.reset() - const selectedPage = websitePages.find(page => page.source_url === selected.id) - setPreviewWebsitePage(selectedPage!) - currentEstimateMutation.mutate() - }} - value={ - { - id: previewWebsitePage?.source_url || '', - name: previewWebsitePage?.title || '', - extension: 'md', - } + +
+
+ {dataSourceType === DataSourceType.FILE && ( + <> +
{t('datasetCreation.stepTwo.fileSource')}
+
+ + {getFileName(files[0].name || '')} + {files.length > 1 && ( + + {t('datasetCreation.stepTwo.other')} + {files.length - 1} + {t('datasetCreation.stepTwo.fileUnit')} + + )} +
+ + )} + {dataSourceType === DataSourceType.NOTION && ( + <> +
{t('datasetCreation.stepTwo.notionSource')}
+
+ + {notionPages[0]?.page_name} + {notionPages.length > 1 && ( + + {t('datasetCreation.stepTwo.other')} + {notionPages.length - 1} + {t('datasetCreation.stepTwo.notionUnit')} + + )} +
+ + )} + {dataSourceType === DataSourceType.WEB && ( + <> +
{t('datasetCreation.stepTwo.websiteSource')}
+
+ + {websitePages[0].source_url} + {websitePages.length > 1 && ( + + {t('datasetCreation.stepTwo.other')} + {websitePages.length - 1} + {t('datasetCreation.stepTwo.webpageUnit')} + + )} +
+ + )} +
+
+
+
{t('datasetCreation.stepTwo.estimateSegment')}
+
+ { + fileIndexingEstimate + ? ( +
{formatNumber(fileIndexingEstimate.total_segments)}
+ ) + : ( +
{t('datasetCreation.stepTwo.calculating')}
+ ) } - /> - } - { - currentDocForm !== ChunkingMode.qa - && - } +
+
- } - className={cn('flex shrink-0 w-1/2 p-4 pr-0 relative h-full', isMobile && 'w-full max-w-[524px]')} - mainClassName='space-y-6' - > - {currentDocForm === ChunkingMode.qa && estimate?.qa_preview && ( - estimate?.qa_preview.map((item, index) => ( - - - - )) - )} - {currentDocForm === ChunkingMode.text && estimate?.preview && ( - estimate?.preview.map((item, index) => ( - - {item.content} - - )) - )} - {currentDocForm === ChunkingMode.parentChild && currentEstimateMutation.data?.preview && ( - estimate?.preview?.map((item, index) => { - const indexForLabel = index + 1 - return ( - - - {item.child_chunks.map((child, index) => { - const indexForLabel = index + 1 - return ( - - ) - })} - - + {!isSetting + ? ( +
+ +
+ +
) - }) - )} - {currentEstimateMutation.isIdle && ( -
-
- -

- {t('datasetCreation.stepTwo.previewChunkTip')} -

+ : ( +
+ + +
+ )} +
+
+
+ + {showPreview &&
+
+
+
+
{t('datasetCreation.stepTwo.previewTitle')}
+ {docForm === DocForm.QA && !previewSwitched && ( + + )} +
+
+
- )} - {currentEstimateMutation.isPending && ( -
- {Array.from({ length: 10 }, (_, i) => ( - - - - - - - - - - - ))} + {docForm === DocForm.QA && !previewSwitched && ( +
+ {t('datasetCreation.stepTwo.previewSwitchTipStart')} + {t('datasetCreation.stepTwo.previewSwitchTipEnd')} +
+ )} +
+
+ {previewSwitched && docForm === DocForm.QA && fileIndexingEstimate?.qa_preview && ( + <> + {fileIndexingEstimate?.qa_preview.map((item, index) => ( + + ))} + + )} + {(docForm === DocForm.TEXT || !previewSwitched) && fileIndexingEstimate?.preview && ( + <> + {fileIndexingEstimate?.preview.map((item, index) => ( + + ))} + + )} + {previewSwitched && docForm === DocForm.QA && !fileIndexingEstimate?.qa_preview && ( +
+ +
+ )} + {!previewSwitched && !fileIndexingEstimate?.preview && ( +
+ +
+ )} +
+
} + {!showPreview && ( +
+
+ +
{t('datasetCreation.stepTwo.sideTipTitle')}
+
+

{t('datasetCreation.stepTwo.sideTipP1')}

+

{t('datasetCreation.stepTwo.sideTipP2')}

+

{t('datasetCreation.stepTwo.sideTipP3')}

+

{t('datasetCreation.stepTwo.sideTipP4')}

+
- )} - +
+ )}
) diff --git a/web/app/components/datasets/create/step-two/inputs.tsx b/web/app/components/datasets/create/step-two/inputs.tsx deleted file mode 100644 index 4231f6242dca20..00000000000000 --- a/web/app/components/datasets/create/step-two/inputs.tsx +++ /dev/null @@ -1,77 +0,0 @@ -import type { FC, PropsWithChildren, ReactNode } from 'react' -import { useTranslation } from 'react-i18next' -import type { InputProps } from '@/app/components/base/input' -import Input from '@/app/components/base/input' -import Tooltip from '@/app/components/base/tooltip' -import type { InputNumberProps } from '@/app/components/base/input-number' -import { InputNumber } from '@/app/components/base/input-number' - -const TextLabel: FC = (props) => { - return -} - -const FormField: FC> = (props) => { - return
- {props.label} - {props.children} -
-} - -export const DelimiterInput: FC = (props) => { - const { t } = useTranslation() - return - {t('datasetCreation.stepTwo.separator')} - - {props.tooltip || t('datasetCreation.stepTwo.separatorTip')} -
- } - /> -
}> - - -} - -export const MaxLengthInput: FC = (props) => { - const { t } = useTranslation() - return - {t('datasetCreation.stepTwo.maxLength')} -
}> - - -} - -export const OverlapInput: FC = (props) => { - const { t } = useTranslation() - return - {t('datasetCreation.stepTwo.overlap')} - - {t('datasetCreation.stepTwo.overlapTip')} -
- } - /> -
}> - - -} diff --git a/web/app/components/datasets/create/step-two/language-select/index.tsx b/web/app/components/datasets/create/step-two/language-select/index.tsx index 9cbf1a40d133fc..41f3e0abb55b6e 100644 --- a/web/app/components/datasets/create/step-two/language-select/index.tsx +++ b/web/app/components/datasets/create/step-two/language-select/index.tsx @@ -1,7 +1,7 @@ 'use client' import type { FC } from 'react' import React from 'react' -import { RiArrowDownSLine, RiCheckLine } from '@remixicon/react' +import { RiArrowDownSLine } from '@remixicon/react' import cn from '@/utils/classnames' import Popover from '@/app/components/base/popover' import { languages } from '@/i18n/language' @@ -22,40 +22,25 @@ const LanguageSelect: FC = ({ manualClose trigger='click' disabled={disabled} - popupClassName='z-20' htmlContent={ -
+
{languages.filter(language => language.supported).map(({ prompt_name }) => (
onSelect(prompt_name)} - > - {prompt_name} - {(currentLanguage === prompt_name) && } + className='py-2 px-3 mx-1 flex items-center gap-2 hover:bg-gray-100 rounded-lg cursor-pointer text-gray-700 text-sm' + onClick={() => onSelect(prompt_name)}>{prompt_name}
))}
} btnElement={ -
- - {currentLanguage} - - +
+ {currentLanguage} +
} - btnClassName={() => cn( - '!border-0 rounded-md !px-1.5 !py-1 !mx-1 !bg-components-button-tertiary-bg !hover:bg-components-button-tertiary-bg', - disabled ? 'bg-components-button-tertiary-bg-disabled' : '', - )} - className='!w-[140px] h-fit !z-20 !translate-x-0 !left-1' + btnClassName={open => cn('!border-0 !px-0 !py-0 !bg-inherit !hover:bg-inherit', open ? 'text-blue-600' : 'text-gray-500')} + className='!w-[120px] h-fit !z-20 !translate-x-0 !left-[-16px]' /> ) } diff --git a/web/app/components/datasets/create/step-two/option-card.tsx b/web/app/components/datasets/create/step-two/option-card.tsx deleted file mode 100644 index b27be757b30d9d..00000000000000 --- a/web/app/components/datasets/create/step-two/option-card.tsx +++ /dev/null @@ -1,101 +0,0 @@ -import { type ComponentProps, type FC, type ReactNode, forwardRef } from 'react' -import Image from 'next/image' -import classNames from '@/utils/classnames' - -const TriangleArrow: FC> = props => ( - - - -) - -type OptionCardHeaderProps = { - icon: ReactNode - title: ReactNode - description: string - isActive?: boolean - activeClassName?: string - effectImg?: string - disabled?: boolean -} - -export const OptionCardHeader: FC = (props) => { - const { icon, title, description, isActive, activeClassName, effectImg, disabled } = props - return
-
- {isActive && effectImg && } -
-
- {icon} -
-
-
- -
-
{title}
-
{description}
-
-
-} - -type OptionCardProps = { - icon: ReactNode - className?: string - activeHeaderClassName?: string - title: ReactNode - description: string - isActive?: boolean - actions?: ReactNode - effectImg?: string - onSwitched?: () => void - noHighlight?: boolean - disabled?: boolean -} & Omit, 'title' | 'onClick'> - -export const OptionCard: FC = forwardRef((props, ref) => { - const { icon, className, title, description, isActive, children, actions, activeHeaderClassName, style, effectImg, onSwitched, noHighlight, disabled, ...rest } = props - return
{ - if (!isActive && !disabled) - onSwitched?.() - }} - {...rest} - ref={ref} - > - - {/** Body */} - {isActive && (children || actions) &&
- {children} - {actions &&
- {actions} -
- } -
} -
-}) - -OptionCard.displayName = 'OptionCard' diff --git a/web/app/components/datasets/create/stepper/index.tsx b/web/app/components/datasets/create/stepper/index.tsx deleted file mode 100644 index 317c1a76eecf57..00000000000000 --- a/web/app/components/datasets/create/stepper/index.tsx +++ /dev/null @@ -1,27 +0,0 @@ -import { type FC, Fragment } from 'react' -import type { Step } from './step' -import { StepperStep } from './step' - -export type StepperProps = { - steps: Step[] - activeIndex: number -} - -export const Stepper: FC = (props) => { - const { steps, activeIndex } = props - return
- {steps.map((step, index) => { - const isLast = index === steps.length - 1 - return ( - - - {!isLast &&
} - - ) - })} -
-} diff --git a/web/app/components/datasets/create/stepper/step.tsx b/web/app/components/datasets/create/stepper/step.tsx deleted file mode 100644 index c230de1a6e748b..00000000000000 --- a/web/app/components/datasets/create/stepper/step.tsx +++ /dev/null @@ -1,46 +0,0 @@ -import type { FC } from 'react' -import classNames from '@/utils/classnames' - -export type Step = { - name: string -} - -export type StepperStepProps = Step & { - index: number - activeIndex: number -} - -export const StepperStep: FC = (props) => { - const { name, activeIndex, index } = props - const isActive = index === activeIndex - const isDisabled = activeIndex < index - const label = isActive ? `STEP ${index + 1}` : `${index + 1}` - return
-
-
- {label} -
-
-
{name}
-
-} diff --git a/web/app/components/datasets/create/top-bar/index.tsx b/web/app/components/datasets/create/top-bar/index.tsx deleted file mode 100644 index 20ba7158db5ed8..00000000000000 --- a/web/app/components/datasets/create/top-bar/index.tsx +++ /dev/null @@ -1,41 +0,0 @@ -import type { FC } from 'react' -import { RiArrowLeftLine } from '@remixicon/react' -import Link from 'next/link' -import { useTranslation } from 'react-i18next' -import { Stepper, type StepperProps } from '../stepper' -import classNames from '@/utils/classnames' - -export type TopbarProps = Pick & { - className?: string -} - -const STEP_T_MAP: Record = { - 1: 'datasetCreation.steps.one', - 2: 'datasetCreation.steps.two', - 3: 'datasetCreation.steps.three', -} - -export const Topbar: FC = (props) => { - const { className, ...rest } = props - const { t } = useTranslation() - return
- -
- -
-

- {t('datasetCreation.steps.header.creation')} -

- -
- ({ - name: t(STEP_T_MAP[i + 1]), - }))} - {...rest} - /> -
-
-} diff --git a/web/app/components/datasets/create/website/base/error-message.tsx b/web/app/components/datasets/create/website/base/error-message.tsx index f061c4624e90f2..aa337ec4bf5323 100644 --- a/web/app/components/datasets/create/website/base/error-message.tsx +++ b/web/app/components/datasets/create/website/base/error-message.tsx @@ -18,7 +18,7 @@ const ErrorMessage: FC = ({ return (
- +
{title}
{errorMsg && ( diff --git a/web/app/components/datasets/create/website/jina-reader/index.tsx b/web/app/components/datasets/create/website/jina-reader/index.tsx index 1c133f935c076b..51d77d712140b7 100644 --- a/web/app/components/datasets/create/website/jina-reader/index.tsx +++ b/web/app/components/datasets/create/website/jina-reader/index.tsx @@ -94,6 +94,7 @@ const JinaReader: FC = ({ const waitForCrawlFinished = useCallback(async (jobId: string) => { try { const res = await checkJinaReaderTaskStatus(jobId) as any + console.log('res', res) if (res.status === 'completed') { return { isError: false, diff --git a/web/app/components/datasets/create/website/preview.tsx b/web/app/components/datasets/create/website/preview.tsx index 5180a834423014..65abe83ed771ac 100644 --- a/web/app/components/datasets/create/website/preview.tsx +++ b/web/app/components/datasets/create/website/preview.tsx @@ -18,7 +18,7 @@ const WebsitePreview = ({ const { t } = useTranslation() return ( -
+
{t('datasetCreation.stepOne.pagePreview')} @@ -32,7 +32,7 @@ const WebsitePreview = ({
{payload.source_url}
-
{payload.markdown}
+
{payload.markdown}
) diff --git a/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx b/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx index 6602244a480a49..36216aa7c89658 100644 --- a/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx +++ b/web/app/components/datasets/documents/detail/batch-modal/csv-downloader.tsx @@ -7,7 +7,7 @@ import { import { useTranslation } from 'react-i18next' import { useContext } from 'use-context-selector' import { Download02 as DownloadIcon } from '@/app/components/base/icons/src/vender/solid/general' -import { ChunkingMode } from '@/models/datasets' +import { DocForm } from '@/models/datasets' import I18n from '@/context/i18n' import { LanguagesSupported } from '@/i18n/language' @@ -32,18 +32,18 @@ const CSV_TEMPLATE_CN = [ ['内容 2'], ] -const CSVDownload: FC<{ docForm: ChunkingMode }> = ({ docForm }) => { +const CSVDownload: FC<{ docForm: DocForm }> = ({ docForm }) => { const { t } = useTranslation() const { locale } = useContext(I18n) const { CSVDownloader, Type } = useCSVDownloader() const getTemplate = () => { if (locale === LanguagesSupported[1]) { - if (docForm === ChunkingMode.qa) + if (docForm === DocForm.QA) return CSV_TEMPLATE_QA_CN return CSV_TEMPLATE_CN } - if (docForm === ChunkingMode.qa) + if (docForm === DocForm.QA) return CSV_TEMPLATE_QA_EN return CSV_TEMPLATE_EN } @@ -52,7 +52,7 @@ const CSVDownload: FC<{ docForm: ChunkingMode }> = ({ docForm }) => {
{t('share.generation.csvStructureTitle')}
- {docForm === ChunkingMode.qa && ( + {docForm === DocForm.QA && ( @@ -72,7 +72,7 @@ const CSVDownload: FC<{ docForm: ChunkingMode }> = ({ docForm }) => {
)} - {docForm === ChunkingMode.text && ( + {docForm === DocForm.TEXT && ( @@ -97,7 +97,7 @@ const CSVDownload: FC<{ docForm: ChunkingMode }> = ({ docForm }) => { bom={true} data={getTemplate()} > -
+
{t('datasetDocuments.list.batchModal.template')}
diff --git a/web/app/components/datasets/documents/detail/batch-modal/index.tsx b/web/app/components/datasets/documents/detail/batch-modal/index.tsx index c666ba67152988..139a364cb40292 100644 --- a/web/app/components/datasets/documents/detail/batch-modal/index.tsx +++ b/web/app/components/datasets/documents/detail/batch-modal/index.tsx @@ -7,11 +7,11 @@ import CSVUploader from './csv-uploader' import CSVDownloader from './csv-downloader' import Button from '@/app/components/base/button' import Modal from '@/app/components/base/modal' -import type { ChunkingMode } from '@/models/datasets' +import type { DocForm } from '@/models/datasets' export type IBatchModalProps = { isShow: boolean - docForm: ChunkingMode + docForm: DocForm onCancel: () => void onConfirm: (file: File) => void } diff --git a/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx b/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx new file mode 100644 index 00000000000000..7b510bcf21b626 --- /dev/null +++ b/web/app/components/datasets/documents/detail/completed/InfiniteVirtualList.tsx @@ -0,0 +1,98 @@ +import type { CSSProperties, FC } from 'react' +import React from 'react' +import { FixedSizeList as List } from 'react-window' +import InfiniteLoader from 'react-window-infinite-loader' +import SegmentCard from './SegmentCard' +import s from './style.module.css' +import type { SegmentDetailModel } from '@/models/datasets' + +type IInfiniteVirtualListProps = { + hasNextPage?: boolean // Are there more items to load? (This information comes from the most recent API request.) + isNextPageLoading: boolean // Are we currently loading a page of items? (This may be an in-flight flag in your Redux store for example.) + items: Array // Array of items loaded so far. + loadNextPage: () => Promise // Callback function responsible for loading the next page of items. + onClick: (detail: SegmentDetailModel) => void + onChangeSwitch: (segId: string, enabled: boolean) => Promise + onDelete: (segId: string) => Promise + archived?: boolean + embeddingAvailable: boolean +} + +const InfiniteVirtualList: FC = ({ + hasNextPage, + isNextPageLoading, + items, + loadNextPage, + onClick: onClickCard, + onChangeSwitch, + onDelete, + archived, + embeddingAvailable, +}) => { + // If there are more items to be loaded then add an extra row to hold a loading indicator. + const itemCount = hasNextPage ? items.length + 1 : items.length + + // Only load 1 page of items at a time. + // Pass an empty callback to InfiniteLoader in case it asks us to load more than once. + const loadMoreItems = isNextPageLoading ? () => { } : loadNextPage + + // Every row is loaded except for our loading indicator row. + const isItemLoaded = (index: number) => !hasNextPage || index < items.length + + // Render an item or a loading indicator. + const Item = ({ index, style }: { index: number; style: CSSProperties }) => { + let content + if (!isItemLoaded(index)) { + content = ( + <> + {[1, 2, 3].map(v => ( + + ))} + + ) + } + else { + content = items[index].map(segItem => ( + onClickCard(segItem)} + onChangeSwitch={onChangeSwitch} + onDelete={onDelete} + loading={false} + archived={archived} + embeddingAvailable={embeddingAvailable} + /> + )) + } + + return ( +
+ {content} +
+ ) + } + + return ( + + {({ onItemsRendered, ref }) => ( + + {Item} + + )} + + ) +} +export default InfiniteVirtualList diff --git a/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx b/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx index 264d62b68a4f32..5b76acc9360c69 100644 --- a/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx +++ b/web/app/components/datasets/documents/detail/completed/SegmentCard.tsx @@ -6,9 +6,9 @@ import { RiDeleteBinLine, } from '@remixicon/react' import { StatusItem } from '../../list' -import style from '../../style.module.css' +import { DocumentTitle } from '../index' import s from './style.module.css' -import { SegmentIndexTag } from './common/segment-index-tag' +import { SegmentIndexTag } from './index' import cn from '@/utils/classnames' import Confirm from '@/app/components/base/confirm' import Switch from '@/app/components/base/switch' @@ -31,22 +31,6 @@ const ProgressBar: FC<{ percent: number; loading: boolean }> = ({ percent, loadi ) } -type DocumentTitleProps = { - extension?: string - name?: string - iconCls?: string - textCls?: string - wrapperCls?: string -} - -const DocumentTitle: FC = ({ extension, name, iconCls, textCls, wrapperCls }) => { - const localExtension = extension?.toLowerCase() || name?.split('.')?.pop()?.toLowerCase() - return
-
- {name || '--'} -
-} - export type UsageScene = 'doc' | 'hitTesting' type ISegmentCardProps = { diff --git a/web/app/components/datasets/documents/detail/completed/child-segment-detail.tsx b/web/app/components/datasets/documents/detail/completed/child-segment-detail.tsx deleted file mode 100644 index 085bfddc163415..00000000000000 --- a/web/app/components/datasets/documents/detail/completed/child-segment-detail.tsx +++ /dev/null @@ -1,134 +0,0 @@ -import React, { type FC, useMemo, useState } from 'react' -import { useTranslation } from 'react-i18next' -import { - RiCloseLine, - RiExpandDiagonalLine, -} from '@remixicon/react' -import ActionButtons from './common/action-buttons' -import ChunkContent from './common/chunk-content' -import Dot from './common/dot' -import { SegmentIndexTag } from './common/segment-index-tag' -import { useSegmentListContext } from './index' -import type { ChildChunkDetail, ChunkingMode } from '@/models/datasets' -import { useEventEmitterContextContext } from '@/context/event-emitter' -import { formatNumber } from '@/utils/format' -import classNames from '@/utils/classnames' -import Divider from '@/app/components/base/divider' -import { formatTime } from '@/utils/time' - -type IChildSegmentDetailProps = { - chunkId: string - childChunkInfo?: Partial & { id: string } - onUpdate: (segmentId: string, childChunkId: string, content: string) => void - onCancel: () => void - docForm: ChunkingMode -} - -/** - * Show all the contents of the segment - */ -const ChildSegmentDetail: FC = ({ - chunkId, - childChunkInfo, - onUpdate, - onCancel, - docForm, -}) => { - const { t } = useTranslation() - const [content, setContent] = useState(childChunkInfo?.content || '') - const { eventEmitter } = useEventEmitterContextContext() - const [loading, setLoading] = useState(false) - const fullScreen = useSegmentListContext(s => s.fullScreen) - const toggleFullScreen = useSegmentListContext(s => s.toggleFullScreen) - - eventEmitter?.useSubscription((v) => { - if (v === 'update-child-segment') - setLoading(true) - if (v === 'update-child-segment-done') - setLoading(false) - }) - - const handleCancel = () => { - onCancel() - setContent(childChunkInfo?.content || '') - } - - const handleSave = () => { - onUpdate(chunkId, childChunkInfo?.id || '', content) - } - - const wordCountText = useMemo(() => { - const count = content.length - return `${formatNumber(count)} ${t('datasetDocuments.segment.characters', { count })}` - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [content.length]) - - const EditTimeText = useMemo(() => { - const timeText = formatTime({ - date: (childChunkInfo?.updated_at ?? 0) * 1000, - dateFormat: 'MM/DD/YYYY h:mm:ss', - }) - return `${t('datasetDocuments.segment.editedAt')} ${timeText}` - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [childChunkInfo?.updated_at]) - - return ( -
-
-
-
{t('datasetDocuments.segment.editChildChunk')}
-
- - - {wordCountText} - - - {EditTimeText} - -
-
-
- {fullScreen && ( - <> - - - - )} -
- -
-
- -
-
-
-
-
- setContent(content)} - isEditMode={true} - /> -
-
- {!fullScreen && ( -
- -
- )} -
- ) -} - -export default React.memo(ChildSegmentDetail) diff --git a/web/app/components/datasets/documents/detail/completed/child-segment-list.tsx b/web/app/components/datasets/documents/detail/completed/child-segment-list.tsx deleted file mode 100644 index 1615ea98cf045a..00000000000000 --- a/web/app/components/datasets/documents/detail/completed/child-segment-list.tsx +++ /dev/null @@ -1,195 +0,0 @@ -import { type FC, useMemo, useState } from 'react' -import { RiArrowDownSLine, RiArrowRightSLine } from '@remixicon/react' -import { useTranslation } from 'react-i18next' -import { EditSlice } from '../../../formatted-text/flavours/edit-slice' -import { useDocumentContext } from '../index' -import { FormattedText } from '../../../formatted-text/formatted' -import Empty from './common/empty' -import FullDocListSkeleton from './skeleton/full-doc-list-skeleton' -import { useSegmentListContext } from './index' -import type { ChildChunkDetail } from '@/models/datasets' -import Input from '@/app/components/base/input' -import classNames from '@/utils/classnames' -import Divider from '@/app/components/base/divider' -import { formatNumber } from '@/utils/format' - -type IChildSegmentCardProps = { - childChunks: ChildChunkDetail[] - parentChunkId: string - handleInputChange?: (value: string) => void - handleAddNewChildChunk?: (parentChunkId: string) => void - enabled: boolean - onDelete?: (segId: string, childChunkId: string) => Promise - onClickSlice?: (childChunk: ChildChunkDetail) => void - total?: number - inputValue?: string - onClearFilter?: () => void - isLoading?: boolean - focused?: boolean -} - -const ChildSegmentList: FC = ({ - childChunks, - parentChunkId, - handleInputChange, - handleAddNewChildChunk, - enabled, - onDelete, - onClickSlice, - total, - inputValue, - onClearFilter, - isLoading, - focused = false, -}) => { - const { t } = useTranslation() - const parentMode = useDocumentContext(s => s.parentMode) - const currChildChunk = useSegmentListContext(s => s.currChildChunk) - - const [collapsed, setCollapsed] = useState(true) - - const toggleCollapse = () => { - setCollapsed(!collapsed) - } - - const isParagraphMode = useMemo(() => { - return parentMode === 'paragraph' - }, [parentMode]) - - const isFullDocMode = useMemo(() => { - return parentMode === 'full-doc' - }, [parentMode]) - - const contentOpacity = useMemo(() => { - return (enabled || focused) ? '' : 'opacity-50 group-hover/card:opacity-100' - }, [enabled, focused]) - - const totalText = useMemo(() => { - const isSearch = inputValue !== '' && isFullDocMode - if (!isSearch) { - const text = isFullDocMode - ? !total - ? '--' - : formatNumber(total) - : formatNumber(childChunks.length) - const count = isFullDocMode - ? text === '--' - ? 0 - : total - : childChunks.length - return `${text} ${t('datasetDocuments.segment.childChunks', { count })}` - } - else { - const text = !total ? '--' : formatNumber(total) - const count = text === '--' ? 0 : total - return `${count} ${t('datasetDocuments.segment.searchResults', { count })}` - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [isFullDocMode, total, childChunks.length, inputValue]) - - return ( -
- {isFullDocMode ? : null} -
-
{ - event.stopPropagation() - toggleCollapse() - }} - > - { - isParagraphMode - ? collapsed - ? ( - - ) - : () - : null - } - {totalText} - · - -
- {isFullDocMode - ? handleInputChange?.(e.target.value)} - onClear={() => handleInputChange?.('')} - /> - : null} -
- {isLoading ? : null} - {((isFullDocMode && !isLoading) || !collapsed) - ?
- {isParagraphMode && ( -
- -
- )} - {childChunks.length > 0 - ? - {childChunks.map((childChunk) => { - const edited = childChunk.updated_at !== childChunk.created_at - const focused = currChildChunk?.childChunkInfo?.id === childChunk.id - return onDelete?.(childChunk.segment_id, childChunk.id)} - labelClassName={focused ? 'bg-state-accent-solid text-text-primary-on-surface' : ''} - labelInnerClassName={'text-[10px] font-semibold align-bottom leading-6'} - contentClassName={classNames('!leading-6', focused ? 'bg-state-accent-hover-alt text-text-primary' : '')} - showDivider={false} - onClick={(e) => { - e.stopPropagation() - onClickSlice?.(childChunk) - }} - offsetOptions={({ rects }) => { - return { - mainAxis: isFullDocMode ? -rects.floating.width : 12 - rects.floating.width, - crossAxis: (20 - rects.floating.height) / 2, - } - }} - /> - })} - - : inputValue !== '' - ?
- -
- : null - } -
- : null} -
- ) -} - -export default ChildSegmentList diff --git a/web/app/components/datasets/documents/detail/completed/common/action-buttons.tsx b/web/app/components/datasets/documents/detail/completed/common/action-buttons.tsx deleted file mode 100644 index 1238d98a9c5025..00000000000000 --- a/web/app/components/datasets/documents/detail/completed/common/action-buttons.tsx +++ /dev/null @@ -1,86 +0,0 @@ -import React, { type FC, useMemo } from 'react' -import { useTranslation } from 'react-i18next' -import { useKeyPress } from 'ahooks' -import { useDocumentContext } from '../../index' -import Button from '@/app/components/base/button' -import { getKeyboardKeyCodeBySystem, getKeyboardKeyNameBySystem } from '@/app/components/workflow/utils' - -type IActionButtonsProps = { - handleCancel: () => void - handleSave: () => void - loading: boolean - actionType?: 'edit' | 'add' - handleRegeneration?: () => void - isChildChunk?: boolean -} - -const ActionButtons: FC = ({ - handleCancel, - handleSave, - loading, - actionType = 'edit', - handleRegeneration, - isChildChunk = false, -}) => { - const { t } = useTranslation() - const mode = useDocumentContext(s => s.mode) - const parentMode = useDocumentContext(s => s.parentMode) - - useKeyPress(['esc'], (e) => { - e.preventDefault() - handleCancel() - }) - - useKeyPress(`${getKeyboardKeyCodeBySystem('ctrl')}.s`, (e) => { - e.preventDefault() - if (loading) - return - handleSave() - } - , { exactMatch: true, useCapture: true }) - - const isParentChildParagraphMode = useMemo(() => { - return mode === 'hierarchical' && parentMode === 'paragraph' - }, [mode, parentMode]) - - return ( -
- - {(isParentChildParagraphMode && actionType === 'edit' && !isChildChunk) - ? - : null - } - -
- ) -} - -ActionButtons.displayName = 'ActionButtons' - -export default React.memo(ActionButtons) diff --git a/web/app/components/datasets/documents/detail/completed/common/add-another.tsx b/web/app/components/datasets/documents/detail/completed/common/add-another.tsx deleted file mode 100644 index 444560e55f7f6b..00000000000000 --- a/web/app/components/datasets/documents/detail/completed/common/add-another.tsx +++ /dev/null @@ -1,32 +0,0 @@ -import React, { type FC } from 'react' -import { useTranslation } from 'react-i18next' -import classNames from '@/utils/classnames' -import Checkbox from '@/app/components/base/checkbox' - -type AddAnotherProps = { - className?: string - isChecked: boolean - onCheck: () => void -} - -const AddAnother: FC = ({ - className, - isChecked, - onCheck, -}) => { - const { t } = useTranslation() - - return ( -
- - {t('datasetDocuments.segment.addAnother')} -
- ) -} - -export default React.memo(AddAnother) diff --git a/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx b/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx deleted file mode 100644 index 3dd3689b64fb43..00000000000000 --- a/web/app/components/datasets/documents/detail/completed/common/batch-action.tsx +++ /dev/null @@ -1,103 +0,0 @@ -import React, { type FC } from 'react' -import { RiArchive2Line, RiCheckboxCircleLine, RiCloseCircleLine, RiDeleteBinLine } from '@remixicon/react' -import { useTranslation } from 'react-i18next' -import { useBoolean } from 'ahooks' -import Divider from '@/app/components/base/divider' -import classNames from '@/utils/classnames' -import Confirm from '@/app/components/base/confirm' - -const i18nPrefix = 'dataset.batchAction' -type IBatchActionProps = { - className?: string - selectedIds: string[] - onBatchEnable: () => void - onBatchDisable: () => void - onBatchDelete: () => Promise - onArchive?: () => void - onCancel: () => void -} - -const BatchAction: FC = ({ - className, - selectedIds, - onBatchEnable, - onBatchDisable, - onArchive, - onBatchDelete, - onCancel, -}) => { - const { t } = useTranslation() - const [isShowDeleteConfirm, { - setTrue: showDeleteConfirm, - setFalse: hideDeleteConfirm, - }] = useBoolean(false) - const [isDeleting, { - setTrue: setIsDeleting, - }] = useBoolean(false) - - const handleBatchDelete = async () => { - setIsDeleting() - await onBatchDelete() - hideDeleteConfirm() - } - return ( -
-
-
- - {selectedIds.length} - - {t(`${i18nPrefix}.selected`)} -
- -
- - -
-
- - -
- {onArchive && ( -
- - -
- )} -
- - -
- - - -
- { - isShowDeleteConfirm && ( - - ) - } -
- ) -} - -export default React.memo(BatchAction) diff --git a/web/app/components/datasets/documents/detail/completed/common/chunk-content.tsx b/web/app/components/datasets/documents/detail/completed/common/chunk-content.tsx deleted file mode 100644 index e6403fa12fd0aa..00000000000000 --- a/web/app/components/datasets/documents/detail/completed/common/chunk-content.tsx +++ /dev/null @@ -1,192 +0,0 @@ -import React, { useEffect, useRef, useState } from 'react' -import type { ComponentProps, FC } from 'react' -import { useTranslation } from 'react-i18next' -import { ChunkingMode } from '@/models/datasets' -import classNames from '@/utils/classnames' - -type IContentProps = ComponentProps<'textarea'> - -const Textarea: FC = React.memo(({ - value, - placeholder, - className, - disabled, - ...rest -}) => { - return ( -