From 2a14c67edcf76034346d55251ed92b4e0eaa366e Mon Sep 17 00:00:00 2001 From: ybalbert001 <120714773+ybalbert001@users.noreply.github.com> Date: Tue, 7 Jan 2025 19:51:23 +0800 Subject: [PATCH 01/42] =?UTF-8?q?Fix=20#12448=20-=20update=20bedrock=20ret?= =?UTF-8?q?rieve=20tool,=20support=20hybrid=20search=20type=20and=20re?= =?UTF-8?q?=E2=80=A6=20(#12446)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Yuanbo Li --- .../builtin/aws/tools/bedrock_retrieve.py | 39 ++++++++++++-- .../builtin/aws/tools/bedrock_retrieve.yaml | 51 +++++++++++++++++++ 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/api/core/tools/provider/builtin/aws/tools/bedrock_retrieve.py b/api/core/tools/provider/builtin/aws/tools/bedrock_retrieve.py index aca369b4389787..2e6a9740c27486 100644 --- a/api/core/tools/provider/builtin/aws/tools/bedrock_retrieve.py +++ b/api/core/tools/provider/builtin/aws/tools/bedrock_retrieve.py @@ -14,14 +14,38 @@ class BedrockRetrieveTool(BuiltinTool): topk: int = None def _bedrock_retrieve( - self, query_input: str, knowledge_base_id: str, num_results: int, metadata_filter: Optional[dict] = None + self, + query_input: str, + knowledge_base_id: str, + num_results: int, + search_type: str, + rerank_model_id: str, + metadata_filter: Optional[dict] = None, ): try: retrieval_query = {"text": query_input} - retrieval_configuration = {"vectorSearchConfiguration": {"numberOfResults": num_results}} + if search_type not in ["HYBRID", "SEMANTIC"]: + raise RuntimeException("search_type should be HYBRID or SEMANTIC") + + retrieval_configuration = { + "vectorSearchConfiguration": {"numberOfResults": num_results, "overrideSearchType": search_type} + } + + if rerank_model_id != "default": + model_for_rerank_arn = f"arn:aws:bedrock:us-west-2::foundation-model/{rerank_model_id}" + rerankingConfiguration = { + "bedrockRerankingConfiguration": { + "numberOfRerankedResults": num_results, + "modelConfiguration": {"modelArn": model_for_rerank_arn}, + }, + "type": "BEDROCK_RERANKING_MODEL", + } - # Add metadata filter to retrieval configuration if present + retrieval_configuration["vectorSearchConfiguration"]["rerankingConfiguration"] = rerankingConfiguration + retrieval_configuration["vectorSearchConfiguration"]["numberOfResults"] = num_results * 5 + + # 如果有元数据过滤条件，则添加到检索配置中 if metadata_filter: retrieval_configuration["vectorSearchConfiguration"]["filter"] = metadata_filter @@ -77,15 +101,20 @@ def _invoke( if not query: return self.create_text_message("Please input query") - # Get metadata filter conditions (if they exist) + # 获取元数据过滤条件（如果存在） metadata_filter_str = tool_parameters.get("metadata_filter") metadata_filter = json.loads(metadata_filter_str) if metadata_filter_str else None + search_type = tool_parameters.get("search_type") + rerank_model_id = tool_parameters.get("rerank_model_id") + line = 4 retrieved_docs = self._bedrock_retrieve( query_input=query, knowledge_base_id=self.knowledge_base_id, num_results=self.topk, + search_type=search_type, + rerank_model_id=rerank_model_id, metadata_filter=metadata_filter, ) @@ -109,7 +138,7 @@ def validate_parameters(self, parameters: dict[str, Any]) -> None: if not parameters.get("query"): raise ValueError("query is required") - # Optional: Validate if metadata filter is a valid JSON string (if provided) + # 可选：可以验证元数据过滤条件是否为有效的 JSON 字符串（如果提供） metadata_filter_str = parameters.get("metadata_filter") if metadata_filter_str and not isinstance(json.loads(metadata_filter_str), dict): raise ValueError("metadata_filter must be a valid JSON object") diff --git a/api/core/tools/provider/builtin/aws/tools/bedrock_retrieve.yaml b/api/core/tools/provider/builtin/aws/tools/bedrock_retrieve.yaml index 31961a0cf03024..f8d1d1d49d3ead 100644 --- a/api/core/tools/provider/builtin/aws/tools/bedrock_retrieve.yaml +++ b/api/core/tools/provider/builtin/aws/tools/bedrock_retrieve.yaml @@ -59,6 +59,57 @@ parameters: max: 10 default: 5 + - name: search_type + type: select + required: false + label: + en_US: search type + zh_Hans: 搜索类型 + pt_BR: search type + human_description: + en_US: search type + zh_Hans: 搜索类型 + pt_BR: search type + llm_description: search type + default: SEMANTIC + options: + - value: SEMANTIC + label: + en_US: SEMANTIC + zh_Hans: 语义搜索 + - value: HYBRID + label: + en_US: HYBRID + zh_Hans: 混合搜索 + form: form + + - name: rerank_model_id + type: select + required: false + label: + en_US: rerank model id + zh_Hans: 重拍模型ID + pt_BR: rerank model id + human_description: + en_US: rerank model id + zh_Hans: 重拍模型ID + pt_BR: rerank model id + llm_description: rerank model id + options: + - value: default + label: + en_US: default + zh_Hans: 默认 + - value: cohere.rerank-v3-5:0 + label: + en_US: cohere.rerank-v3-5:0 + zh_Hans: cohere.rerank-v3-5:0 + - value: amazon.rerank-v1:0 + label: + en_US: amazon.rerank-v1:0 + zh_Hans: amazon.rerank-v1:0 + form: form + - name: aws_region type: string required: false From 4e6c86341d6e7ab2e80cb5a6f849aa282f8bad12 Mon Sep 17 00:00:00 2001 From: Infinitnet <6189915+infinitnet@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:51:38 +0100 Subject: [PATCH 02/42] Add 'document' feature to Sonnet 3.5 through OpenRouter (#12444) --- .../model_providers/openrouter/llm/claude-3-5-sonnet.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/api/core/model_runtime/model_providers/openrouter/llm/claude-3-5-sonnet.yaml b/api/core/model_runtime/model_providers/openrouter/llm/claude-3-5-sonnet.yaml index e829048e55e0d8..caed5a901db853 100644 --- a/api/core/model_runtime/model_providers/openrouter/llm/claude-3-5-sonnet.yaml +++ b/api/core/model_runtime/model_providers/openrouter/llm/claude-3-5-sonnet.yaml @@ -7,6 +7,7 @@ features: - vision - tool-call - stream-tool-call + - document model_properties: mode: chat context_size: 200000 From fd2bfff023ff025b1e9c324512564d9b91bdf9aa Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Tue, 7 Jan 2025 21:30:23 +0800 Subject: [PATCH 03/42] remove knowledge admin role (#12450) --- api/services/dataset_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 701261d7cbd494..3be8a38b031d9c 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -86,7 +86,7 @@ def get_datasets(page, per_page, tenant_id=None, user=None, search=None, tag_ids else: return [], 0 else: - if user.current_role not in (TenantAccountRole.OWNER, TenantAccountRole.ADMIN): + if user.current_role != TenantAccountRole.OWNER: # show all datasets that the user has permission to access if permitted_dataset_ids: query = query.filter( @@ -382,7 +382,7 @@ def check_dataset_permission(dataset, user): if dataset.tenant_id != user.current_tenant_id: logging.debug(f"User {user.id} does not have permission to access dataset {dataset.id}") raise NoPermissionError("You do not have permission to access this dataset.") - if user.current_role not in (TenantAccountRole.OWNER, TenantAccountRole.ADMIN): + if user.current_role != TenantAccountRole.OWNER: if dataset.permission == DatasetPermissionEnum.ONLY_ME and dataset.created_by != user.id: logging.debug(f"User {user.id} does not have permission to access dataset {dataset.id}") raise NoPermissionError("You do not have permission to access this dataset.") @@ -404,7 +404,7 @@ def check_dataset_operator_permission(user: Optional[Account] = None, dataset: O if not user: raise ValueError("User not found") - if user.current_role not in (TenantAccountRole.OWNER, TenantAccountRole.ADMIN): + if user.current_role != TenantAccountRole.OWNER: if dataset.permission == DatasetPermissionEnum.ONLY_ME: if dataset.created_by != user.id: raise NoPermissionError("You do not have permission to access this dataset.") From 67228c9b267ed6c85044c5bdf74618731cbc1a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?= Date: Tue, 7 Jan 2025 21:55:51 +0800 Subject: [PATCH 04/42] fix: url with variable not work (#12452) --- api/core/workflow/nodes/http_request/executor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/core/workflow/nodes/http_request/executor.py b/api/core/workflow/nodes/http_request/executor.py index 941a0cb078be25..87b71394e4f4a1 100644 --- a/api/core/workflow/nodes/http_request/executor.py +++ b/api/core/workflow/nodes/http_request/executor.py @@ -82,12 +82,6 @@ def __init__( node_data.authorization.config.api_key ).text - # check if node_data.url is a valid URL - if not node_data.url: - raise InvalidURLError("url is required") - if not node_data.url.startswith(("http://", "https://")): - raise InvalidURLError("url should start with http:// or https://") - self.url: str = node_data.url self.method = node_data.method self.auth = node_data.authorization @@ -114,6 +108,12 @@ def _initialize(self): def _init_url(self): self.url = self.variable_pool.convert_template(self.node_data.url).text + # check if url is a valid URL + if not self.url: + raise InvalidURLError("url is required") + if not self.url.startswith(("http://", "https://")): + raise InvalidURLError("url should start with http:// or https://") + def _init_params(self): """ Almost same as _init_headers(), difference: From 4295cefeb11cf1ab1b3f84eac94612f43f62f07f Mon Sep 17 00:00:00 2001 From: Hiroshi Fujita Date: Tue, 7 Jan 2025 23:33:25 +0900 Subject: [PATCH 05/42] fix: allow fallback to remote_url when url is not provided (#12455) --- api/factories/file_factory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/factories/file_factory.py b/api/factories/file_factory.py index 1e1e3fb79642f4..c6dc748e93bd03 100644 --- a/api/factories/file_factory.py +++ b/api/factories/file_factory.py @@ -158,7 +158,7 @@ def _build_from_remote_url( tenant_id: str, transfer_method: FileTransferMethod, ) -> File: - url = mapping.get("url") + url = mapping.get("url") or mapping.get("remote_url") if not url: raise ValueError("Invalid file url") From 05bda6f38df8bffef89208196f3756c4aabd9448 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Wed, 8 Jan 2025 08:55:44 +0800 Subject: [PATCH 06/42] add tidb on qdrant redis lock (#12462) --- .../tidb_on_qdrant/tidb_on_qdrant_vector.py | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py b/api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py index 19c5579a688f5a..549f0175eb84c4 100644 --- a/api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py +++ b/api/core/rag/datasource/vdb/tidb_on_qdrant/tidb_on_qdrant_vector.py @@ -409,27 +409,27 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings db.session.query(TidbAuthBinding).filter(TidbAuthBinding.tenant_id == dataset.tenant_id).one_or_none() ) if not tidb_auth_binding: - idle_tidb_auth_binding = ( - db.session.query(TidbAuthBinding) - .filter(TidbAuthBinding.active == False, TidbAuthBinding.status == "ACTIVE") - .limit(1) - .one_or_none() - ) - if idle_tidb_auth_binding: - idle_tidb_auth_binding.active = True - idle_tidb_auth_binding.tenant_id = dataset.tenant_id - db.session.commit() - TIDB_ON_QDRANT_API_KEY = f"{idle_tidb_auth_binding.account}:{idle_tidb_auth_binding.password}" - else: - with redis_client.lock("create_tidb_serverless_cluster_lock", timeout=900): - tidb_auth_binding = ( + with redis_client.lock("create_tidb_serverless_cluster_lock", timeout=900): + tidb_auth_binding = ( + db.session.query(TidbAuthBinding) + .filter(TidbAuthBinding.tenant_id == dataset.tenant_id) + .one_or_none() + ) + if tidb_auth_binding: + TIDB_ON_QDRANT_API_KEY = f"{tidb_auth_binding.account}:{tidb_auth_binding.password}" + + else: + idle_tidb_auth_binding = ( db.session.query(TidbAuthBinding) - .filter(TidbAuthBinding.tenant_id == dataset.tenant_id) + .filter(TidbAuthBinding.active == False, TidbAuthBinding.status == "ACTIVE") + .limit(1) .one_or_none() ) - if tidb_auth_binding: - TIDB_ON_QDRANT_API_KEY = f"{tidb_auth_binding.account}:{tidb_auth_binding.password}" - + if idle_tidb_auth_binding: + idle_tidb_auth_binding.active = True + idle_tidb_auth_binding.tenant_id = dataset.tenant_id + db.session.commit() + TIDB_ON_QDRANT_API_KEY = f"{idle_tidb_auth_binding.account}:{idle_tidb_auth_binding.password}" else: new_cluster = TidbService.create_tidb_serverless_cluster( dify_config.TIDB_PROJECT_ID or "", @@ -451,7 +451,6 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings db.session.add(new_tidb_auth_binding) db.session.commit() TIDB_ON_QDRANT_API_KEY = f"{new_tidb_auth_binding.account}:{new_tidb_auth_binding.password}" - else: TIDB_ON_QDRANT_API_KEY = f"{tidb_auth_binding.account}:{tidb_auth_binding.password}" From 6222179a57144007ce49b652f84288d6e8eae02f Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Wed, 8 Jan 2025 10:50:34 +0800 Subject: [PATCH 07/42] Revert "fix:deepseek tool call not working correctly" (#12463) --- .../model_providers/openai_api_compatible/llm/llm.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py index 2fd0c4ec1fbd42..734cf28b1578f4 100644 --- a/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py +++ b/api/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py @@ -377,10 +377,7 @@ def _generate( for tool in tools: formatted_tools.append(helper.dump_model(PromptMessageFunction(function=tool))) - if prompt_messages[-1].role.value == "tool": - data["tools"] = None - else: - data["tools"] = formatted_tools + data["tools"] = formatted_tools if stop: data["stop"] = stop From 6635c393e931e9c29ea6c7b884fab233fe4e7f6e Mon Sep 17 00:00:00 2001 From: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Date: Wed, 8 Jan 2025 12:11:45 +0800 Subject: [PATCH 08/42] fix: adjust opacity for model selector based on readonly state (#12472) --- web/app/components/datasets/create/step-two/index.tsx | 1 + .../model-provider-page/model-selector/model-trigger.tsx | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 8353467bec6298..27ca16579bcc56 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -932,6 +932,7 @@ const StepTwo = ({
{t('datasetSettings.form.embeddingModel')}
{ diff --git a/web/app/components/header/account-setting/model-provider-page/model-selector/model-trigger.tsx b/web/app/components/header/account-setting/model-provider-page/model-selector/model-trigger.tsx index aba7ff5f64599f..556a2ef66f33a3 100644 --- a/web/app/components/header/account-setting/model-provider-page/model-selector/model-trigger.tsx +++ b/web/app/components/header/account-setting/model-provider-page/model-selector/model-trigger.tsx @@ -36,7 +36,6 @@ const ModelTrigger: FC = ({ className={classNames( 'group flex items-center px-2 h-8 rounded-lg bg-components-input-bg-normal', !readonly && 'hover:bg-components-input-bg-hover cursor-pointer', - !!readonly && 'opacity-50', className, open && '!bg-components-input-bg-hover', model.status !== ModelStatusEnum.active && '!bg-[#FFFAEB]', From d2586278d6dc0859b93326029e87c1af68bba851 Mon Sep 17 00:00:00 2001 From: Hiroshi Fujita Date: Wed, 8 Jan 2025 13:35:41 +0900 Subject: [PATCH 09/42] Feat elasticsearch japanese (#12194) --- api/controllers/console/datasets/datasets.py | 2 + .../elasticsearch/elasticsearch_ja_vector.py | 104 ++++++++++++++++++ api/core/rag/datasource/vdb/vector_factory.py | 6 + api/core/rag/datasource/vdb/vector_type.py | 1 + docker/.env.example | 4 +- docker/docker-compose.yaml | 10 +- docker/elasticsearch/docker-entrypoint.sh | 25 +++++ 7 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 api/core/rag/datasource/vdb/elasticsearch/elasticsearch_ja_vector.py create mode 100755 docker/elasticsearch/docker-entrypoint.sh diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 0c0d2e20035b43..2da45a7bb67bc6 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -640,6 +640,7 @@ def get(self): | VectorType.MYSCALE | VectorType.ORACLE | VectorType.ELASTICSEARCH + | VectorType.ELASTICSEARCH_JA | VectorType.PGVECTOR | VectorType.TIDB_ON_QDRANT | VectorType.LINDORM @@ -683,6 +684,7 @@ def get(self, vector_type): | VectorType.MYSCALE | VectorType.ORACLE | VectorType.ELASTICSEARCH + | VectorType.ELASTICSEARCH_JA | VectorType.COUCHBASE | VectorType.PGVECTOR | VectorType.LINDORM diff --git a/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_ja_vector.py b/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_ja_vector.py new file mode 100644 index 00000000000000..27575197faccfe --- /dev/null +++ b/api/core/rag/datasource/vdb/elasticsearch/elasticsearch_ja_vector.py @@ -0,0 +1,104 @@ +import json +import logging +from typing import Any, Optional + +from flask import current_app + +from core.rag.datasource.vdb.elasticsearch.elasticsearch_vector import ( + ElasticSearchConfig, + ElasticSearchVector, + ElasticSearchVectorFactory, +) +from core.rag.datasource.vdb.field import Field +from core.rag.datasource.vdb.vector_type import VectorType +from core.rag.embedding.embedding_base import Embeddings +from extensions.ext_redis import redis_client +from models.dataset import Dataset + +logger = logging.getLogger(__name__) + + +class ElasticSearchJaVector(ElasticSearchVector): + def create_collection( + self, + embeddings: list[list[float]], + metadatas: Optional[list[dict[Any, Any]]] = None, + index_params: Optional[dict] = None, + ): + lock_name = f"vector_indexing_lock_{self._collection_name}" + with redis_client.lock(lock_name, timeout=20): + collection_exist_cache_key = f"vector_indexing_{self._collection_name}" + if redis_client.get(collection_exist_cache_key): + logger.info(f"Collection {self._collection_name} already exists.") + return + + if not self._client.indices.exists(index=self._collection_name): + dim = len(embeddings[0]) + settings = { + "analysis": { + "analyzer": { + "ja_analyzer": { + "type": "custom", + "char_filter": [ + "icu_normalizer", + "kuromoji_iteration_mark", + ], + "tokenizer": "kuromoji_tokenizer", + "filter": [ + "kuromoji_baseform", + "kuromoji_part_of_speech", + "ja_stop", + "kuromoji_number", + "kuromoji_stemmer", + ], + } + } + } + } + mappings = { + "properties": { + Field.CONTENT_KEY.value: { + "type": "text", + "analyzer": "ja_analyzer", + "search_analyzer": "ja_analyzer", + }, + Field.VECTOR.value: { # Make sure the dimension is correct here + "type": "dense_vector", + "dims": dim, + "index": True, + "similarity": "cosine", + }, + Field.METADATA_KEY.value: { + "type": "object", + "properties": { + "doc_id": {"type": "keyword"} # Map doc_id to keyword type + }, + }, + } + } + self._client.indices.create(index=self._collection_name, settings=settings, mappings=mappings) + + redis_client.set(collection_exist_cache_key, 1, ex=3600) + + +class ElasticSearchJaVectorFactory(ElasticSearchVectorFactory): + def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> ElasticSearchJaVector: + if dataset.index_struct_dict: + class_prefix: str = dataset.index_struct_dict["vector_store"]["class_prefix"] + collection_name = class_prefix + else: + dataset_id = dataset.id + collection_name = Dataset.gen_collection_name_by_id(dataset_id) + dataset.index_struct = json.dumps(self.gen_index_struct_dict(VectorType.ELASTICSEARCH, collection_name)) + + config = current_app.config + return ElasticSearchJaVector( + index_name=collection_name, + config=ElasticSearchConfig( + host=config.get("ELASTICSEARCH_HOST", "localhost"), + port=config.get("ELASTICSEARCH_PORT", 9200), + username=config.get("ELASTICSEARCH_USERNAME", ""), + password=config.get("ELASTICSEARCH_PASSWORD", ""), + ), + attributes=[], + ) diff --git a/api/core/rag/datasource/vdb/vector_factory.py b/api/core/rag/datasource/vdb/vector_factory.py index 523fa80f124b0c..bdc40e29c7e838 100644 --- a/api/core/rag/datasource/vdb/vector_factory.py +++ b/api/core/rag/datasource/vdb/vector_factory.py @@ -90,6 +90,12 @@ def get_vector_factory(vector_type: str) -> type[AbstractVectorFactory]: from core.rag.datasource.vdb.elasticsearch.elasticsearch_vector import ElasticSearchVectorFactory return ElasticSearchVectorFactory + case VectorType.ELASTICSEARCH_JA: + from core.rag.datasource.vdb.elasticsearch.elasticsearch_ja_vector import ( + ElasticSearchJaVectorFactory, + ) + + return ElasticSearchJaVectorFactory case VectorType.TIDB_VECTOR: from core.rag.datasource.vdb.tidb_vector.tidb_vector import TiDBVectorFactory diff --git a/api/core/rag/datasource/vdb/vector_type.py b/api/core/rag/datasource/vdb/vector_type.py index 05183c03717fa3..e73411aa0d38a9 100644 --- a/api/core/rag/datasource/vdb/vector_type.py +++ b/api/core/rag/datasource/vdb/vector_type.py @@ -16,6 +16,7 @@ class VectorType(StrEnum): TENCENT = "tencent" ORACLE = "oracle" ELASTICSEARCH = "elasticsearch" + ELASTICSEARCH_JA = "elasticsearch-ja" LINDORM = "lindorm" COUCHBASE = "couchbase" BAIDU = "baidu" diff --git a/docker/.env.example b/docker/.env.example index 7c5447ef5bbf17..85277027b663c2 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -383,7 +383,7 @@ SUPABASE_URL=your-server-url # ------------------------------ # The type of vector store to use. -# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`, `elasticsearch`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`. +# Supported values are `weaviate`, `qdrant`, `milvus`, `myscale`, `relyt`, `pgvector`, `pgvecto-rs`, `chroma`, `opensearch`, `tidb_vector`, `oracle`, `tencent`, `elasticsearch`, `elasticsearch-ja`, `analyticdb`, `couchbase`, `vikingdb`, `oceanbase`. VECTOR_STORE=weaviate # The Weaviate endpoint URL. Only available when VECTOR_STORE is `weaviate`. @@ -512,7 +512,7 @@ TENCENT_VECTOR_DB_SHARD=1 TENCENT_VECTOR_DB_REPLICAS=2 # ElasticSearch configuration, only available when VECTOR_STORE is `elasticsearch` -ELASTICSEARCH_HOST=0.0.0.0 +ELASTICSEARCH_HOST=elasticsearch ELASTICSEARCH_PORT=9200 ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=elastic diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index a24d4fbbd0d477..29e312de7f9c8b 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -883,20 +883,28 @@ services: container_name: elasticsearch profiles: - elasticsearch + - elasticsearch-ja restart: always volumes: + - ./elasticsearch/docker-entrypoint.sh:/docker-entrypoint-mount.sh - dify_es01_data:/usr/share/elasticsearch/data environment: ELASTIC_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic} + VECTOR_STORE: ${VECTOR_STORE:-} cluster.name: dify-es-cluster node.name: dify-es0 discovery.type: single-node - xpack.license.self_generated.type: trial + xpack.license.self_generated.type: basic xpack.security.enabled: 'true' xpack.security.enrollment.enabled: 'false' xpack.security.http.ssl.enabled: 'false' ports: - ${ELASTICSEARCH_PORT:-9200}:9200 + deploy: + resources: + limits: + memory: 2g + entrypoint: [ 'sh', '-c', "sh /docker-entrypoint-mount.sh" ] healthcheck: test: [ 'CMD', 'curl', '-s', 'http://localhost:9200/_cluster/health?pretty' ] interval: 30s diff --git a/docker/elasticsearch/docker-entrypoint.sh b/docker/elasticsearch/docker-entrypoint.sh new file mode 100755 index 00000000000000..6669aec5a921c2 --- /dev/null +++ b/docker/elasticsearch/docker-entrypoint.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e + +if [ "${VECTOR_STORE}" = "elasticsearch-ja" ]; then + # Check if the ICU tokenizer plugin is installed + if ! /usr/share/elasticsearch/bin/elasticsearch-plugin list | grep -q analysis-icu; then + printf '%s\n' "Installing the ICU tokenizer plugin" + if ! /usr/share/elasticsearch/bin/elasticsearch-plugin install analysis-icu; then + printf '%s\n' "Failed to install the ICU tokenizer plugin" + exit 1 + fi + fi + # Check if the Japanese language analyzer plugin is installed + if ! /usr/share/elasticsearch/bin/elasticsearch-plugin list | grep -q analysis-kuromoji; then + printf '%s\n' "Installing the Japanese language analyzer plugin" + if ! /usr/share/elasticsearch/bin/elasticsearch-plugin install analysis-kuromoji; then + printf '%s\n' "Failed to install the Japanese language analyzer plugin" + exit 1 + fi + fi +fi + +# Run the original entrypoint script +exec /bin/tini -- /usr/local/bin/docker-entrypoint.sh From 53bb37b749532efce5bb743143279008c61a4b3d Mon Sep 17 00:00:00 2001 From: Yingchun Lai Date: Wed, 8 Jan 2025 12:52:45 +0800 Subject: [PATCH 10/42] fix: fix the incorrect plaintext file key when saving (#10429) --- api/core/rag/extractor/pdf_extractor.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/core/rag/extractor/pdf_extractor.py b/api/core/rag/extractor/pdf_extractor.py index 89a7061c26accc..04033dec3fa56d 100644 --- a/api/core/rag/extractor/pdf_extractor.py +++ b/api/core/rag/extractor/pdf_extractor.py @@ -23,7 +23,6 @@ def __init__(self, file_path: str, file_cache_key: Optional[str] = None): self._file_cache_key = file_cache_key def extract(self) -> list[Document]: - plaintext_file_key = "" plaintext_file_exists = False if self._file_cache_key: try: @@ -39,8 +38,8 @@ def extract(self) -> list[Document]: text = "\n\n".join(text_list) # save plaintext file for caching - if not plaintext_file_exists and plaintext_file_key: - storage.save(plaintext_file_key, text.encode("utf-8")) + if not plaintext_file_exists and self._file_cache_key: + storage.save(self._file_cache_key, text.encode("utf-8")) return documents From 0a49d3dd5274e1f37d9d05ccbc7284c64d9d20a4 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Wed, 8 Jan 2025 14:49:44 +0800 Subject: [PATCH 11/42] fix: tiktoken cannot be loaded without internet (#12478) Signed-off-by: -LAN- --- .../__base/tokenizers/gpt2_tokenzier.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py index 9a5c40addb1ec8..7f3c4a61e45b06 100644 --- a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py +++ b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py @@ -1,8 +1,6 @@ from threading import Lock from typing import Any -import tiktoken - _tokenizer: Any = None _lock = Lock() @@ -33,9 +31,17 @@ def get_encoder() -> Any: if _tokenizer is None: # Try to use tiktoken to get the tokenizer because it is faster # - _tokenizer = tiktoken.get_encoding("gpt2") - # base_path = abspath(__file__) - # gpt2_tokenizer_path = join(dirname(base_path), "gpt2") - # _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path) + try: + import tiktoken + + _tokenizer = tiktoken.get_encoding("gpt2") + except Exception: + from os.path import abspath, dirname, join + + from transformers import GPT2Tokenizer as TransformerGPT2Tokenizer # type: ignore + + base_path = abspath(__file__) + gpt2_tokenizer_path = join(dirname(base_path), "gpt2") + _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path) return _tokenizer From d649037c3e1c3f8542640b37a122622b19ce5475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?= Date: Wed, 8 Jan 2025 15:20:15 +0800 Subject: [PATCH 12/42] feat: support single run doc extractor node (#11318) --- .../workflow/nodes/document_extractor/node.py | 20 ++++++++- .../nodes/_base/hooks/use-one-step-run.ts | 3 ++ .../nodes/document-extractor/panel.tsx | 37 ++++++++++++++- .../nodes/document-extractor/use-config.ts | 45 ++++++++++++++++++- web/app/components/workflow/utils.ts | 1 + 5 files changed, 103 insertions(+), 3 deletions(-) diff --git a/api/core/workflow/nodes/document_extractor/node.py b/api/core/workflow/nodes/document_extractor/node.py index 38b382a7b6ff03..c0d8c6409982e6 100644 --- a/api/core/workflow/nodes/document_extractor/node.py +++ b/api/core/workflow/nodes/document_extractor/node.py @@ -5,7 +5,8 @@ import operator import os import tempfile -from typing import cast +from collections.abc import Mapping, Sequence +from typing import Any, cast import docx import pandas as pd @@ -81,6 +82,23 @@ def _run(self): process_data=process_data, ) + @classmethod + def _extract_variable_selector_to_variable_mapping( + cls, + *, + graph_config: Mapping[str, Any], + node_id: str, + node_data: DocumentExtractorNodeData, + ) -> Mapping[str, Sequence[str]]: + """ + Extract variable selector to variable mapping + :param graph_config: graph config + :param node_id: node id + :param node_data: node data + :return: + """ + return {node_id + ".files": node_data.variable_selector} + def _extract_text_by_mime_type(*, file_content: bytes, mime_type: str) -> str: """Extract text from a file based on its MIME type.""" diff --git a/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts b/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts index 6791a2f746c9d2..05481b4c8cb494 100644 --- a/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts +++ b/web/app/components/workflow/nodes/_base/hooks/use-one-step-run.ts @@ -27,6 +27,7 @@ import VariableAssigner from '@/app/components/workflow/nodes/variable-assigner/ import Assigner from '@/app/components/workflow/nodes/assigner/default' import ParameterExtractorDefault from '@/app/components/workflow/nodes/parameter-extractor/default' import IterationDefault from '@/app/components/workflow/nodes/iteration/default' +import DocumentExtractorDefault from '@/app/components/workflow/nodes/document-extractor/default' import { ssePost } from '@/service/base' import { getInputVars as doGetInputVars } from '@/app/components/base/prompt-editor/constants' @@ -43,6 +44,7 @@ const { checkValid: checkVariableAssignerValid } = VariableAssigner const { checkValid: checkAssignerValid } = Assigner const { checkValid: checkParameterExtractorValid } = ParameterExtractorDefault const { checkValid: checkIterationValid } = IterationDefault +const { checkValid: checkDocumentExtractorValid } = DocumentExtractorDefault const checkValidFns: Record = { [BlockEnum.LLM]: checkLLMValid, @@ -57,6 +59,7 @@ const checkValidFns: Record = { [BlockEnum.VariableAggregator]: checkVariableAssignerValid, [BlockEnum.ParameterExtractor]: checkParameterExtractorValid, [BlockEnum.Iteration]: checkIterationValid, + [BlockEnum.DocExtractor]: checkDocumentExtractorValid, } as any type Params = { diff --git a/web/app/components/workflow/nodes/document-extractor/panel.tsx b/web/app/components/workflow/nodes/document-extractor/panel.tsx index 1e26fe4c337ee2..597f528582e2a0 100644 --- a/web/app/components/workflow/nodes/document-extractor/panel.tsx +++ b/web/app/components/workflow/nodes/document-extractor/panel.tsx @@ -11,9 +11,11 @@ import useConfig from './use-config' import type { DocExtractorNodeType } from './types' import { fetchSupportFileTypes } from '@/service/datasets' import Field from '@/app/components/workflow/nodes/_base/components/field' -import { BlockEnum, type NodePanelProps } from '@/app/components/workflow/types' +import { BlockEnum, InputVarType, type NodePanelProps } from '@/app/components/workflow/types' import I18n from '@/context/i18n' import { LanguagesSupported } from '@/i18n/language' +import BeforeRunForm from '@/app/components/workflow/nodes/_base/components/before-run-form' +import ResultPanel from '@/app/components/workflow/run/result-panel' const i18nPrefix = 'workflow.nodes.docExtractor' @@ -46,6 +48,15 @@ const Panel: FC> = ({ inputs, handleVarChanges, filterVar, + // single run + isShowSingleRun, + hideSingleRun, + runningStatus, + handleRun, + handleStop, + runResult, + files, + setFiles, } = useConfig(id, data) return ( @@ -81,6 +92,30 @@ const Panel: FC> = ({ /> + { + isShowSingleRun && ( + setFiles((keyValue as any).files), + }, + ]} + runningStatus={runningStatus} + onRun={handleRun} + onStop={handleStop} + result={} + /> + ) + } ) } diff --git a/web/app/components/workflow/nodes/document-extractor/use-config.ts b/web/app/components/workflow/nodes/document-extractor/use-config.ts index 9406c125f046f8..9d720d7b6397ff 100644 --- a/web/app/components/workflow/nodes/document-extractor/use-config.ts +++ b/web/app/components/workflow/nodes/document-extractor/use-config.ts @@ -3,9 +3,10 @@ import produce from 'immer' import { useStoreApi } from 'reactflow' import type { ValueSelector, Var } from '../../types' -import { VarType } from '../../types' +import { InputVarType, VarType } from '../../types' import { type DocExtractorNodeType } from './types' import useNodeCrud from '@/app/components/workflow/nodes/_base/hooks/use-node-crud' +import useOneStepRun from '@/app/components/workflow/nodes/_base/hooks/use-one-step-run' import { useIsChatMode, useNodesReadOnly, @@ -55,11 +56,53 @@ const useConfig = (id: string, payload: DocExtractorNodeType) => { setInputs(newInputs) }, [getType, inputs, setInputs]) + // single run + const { + isShowSingleRun, + hideSingleRun, + runningStatus, + isCompleted, + handleRun, + handleStop, + runInputData, + setRunInputData, + runResult, + } = useOneStepRun({ + id, + data: inputs, + defaultRunInputData: { files: [] }, + }) + const varInputs = [{ + label: inputs.title, + variable: 'files', + type: InputVarType.multiFiles, + required: true, + }] + + const files = runInputData.files + const setFiles = useCallback((newFiles: []) => { + setRunInputData({ + ...runInputData, + files: newFiles, + }) + }, [runInputData, setRunInputData]) + return { readOnly, inputs, filterVar, handleVarChanges, + // single run + isShowSingleRun, + hideSingleRun, + runningStatus, + isCompleted, + handleRun, + handleStop, + varInputs, + files, + setFiles, + runResult, } } diff --git a/web/app/components/workflow/utils.ts b/web/app/components/workflow/utils.ts index 4c61267e4cfc2a..da2848277f4db4 100644 --- a/web/app/components/workflow/utils.ts +++ b/web/app/components/workflow/utils.ts @@ -382,6 +382,7 @@ export const canRunBySingle = (nodeType: BlockEnum) => { || nodeType === BlockEnum.Tool || nodeType === BlockEnum.ParameterExtractor || nodeType === BlockEnum.Iteration + || nodeType === BlockEnum.DocExtractor } type ConnectedSourceOrTargetNodesChange = { From 040a3b782cc0bd20a71e839a10e1c18c26014191 Mon Sep 17 00:00:00 2001 From: YoungLH <974840768@qq.com> Date: Wed, 8 Jan 2025 17:39:53 +0800 Subject: [PATCH 13/42] FEAT: support milvus to full text search (#11430) Signed-off-by: YoungLH <974840768@qq.com> --- api/configs/middleware/vdb/milvus_config.py | 6 + api/core/rag/datasource/vdb/field.py | 2 + .../datasource/vdb/milvus/milvus_vector.py | 200 +++++++-- api/poetry.lock | 412 +++++++++--------- api/pyproject.toml | 4 +- .../vdb/milvus/test_milvus.py | 4 +- docker/.env.example | 1 + docker/docker-compose.yaml | 3 +- 8 files changed, 392 insertions(+), 240 deletions(-) diff --git a/api/configs/middleware/vdb/milvus_config.py b/api/configs/middleware/vdb/milvus_config.py index 231cbbbe8ffc9e..ebdf8857b962b7 100644 --- a/api/configs/middleware/vdb/milvus_config.py +++ b/api/configs/middleware/vdb/milvus_config.py @@ -33,3 +33,9 @@ class MilvusConfig(BaseSettings): description="Name of the Milvus database to connect to (default is 'default')", default="default", ) + + MILVUS_ENABLE_HYBRID_SEARCH: bool = Field( + description="Enable hybrid search features (requires Milvus >= 2.5.0). Set to false for compatibility with " + "older versions", + default=True, + ) diff --git a/api/core/rag/datasource/vdb/field.py b/api/core/rag/datasource/vdb/field.py index 1c16e4d9cdacde..a64407bce1c14a 100644 --- a/api/core/rag/datasource/vdb/field.py +++ b/api/core/rag/datasource/vdb/field.py @@ -6,6 +6,8 @@ class Field(Enum): METADATA_KEY = "metadata" GROUP_KEY = "group_id" VECTOR = "vector" + # Sparse Vector aims to support full text search + SPARSE_VECTOR = "sparse_vector" TEXT_KEY = "text" PRIMARY_KEY = "id" DOC_ID = "metadata.doc_id" diff --git a/api/core/rag/datasource/vdb/milvus/milvus_vector.py b/api/core/rag/datasource/vdb/milvus/milvus_vector.py index 9b029ffc193cc0..9a184f7dd99ad9 100644 --- a/api/core/rag/datasource/vdb/milvus/milvus_vector.py +++ b/api/core/rag/datasource/vdb/milvus/milvus_vector.py @@ -2,6 +2,7 @@ import logging from typing import Any, Optional +from packaging import version from pydantic import BaseModel, model_validator from pymilvus import MilvusClient, MilvusException # type: ignore from pymilvus.milvus_client import IndexParams # type: ignore @@ -20,16 +21,25 @@ class MilvusConfig(BaseModel): - uri: str - token: Optional[str] = None - user: str - password: str - batch_size: int = 100 - database: str = "default" + """ + Configuration class for Milvus connection. + """ + + uri: str # Milvus server URI + token: Optional[str] = None # Optional token for authentication + user: str # Username for authentication + password: str # Password for authentication + batch_size: int = 100 # Batch size for operations + database: str = "default" # Database name + enable_hybrid_search: bool = False # Flag to enable hybrid search @model_validator(mode="before") @classmethod def validate_config(cls, values: dict) -> dict: + """ + Validate the configuration values. + Raises ValueError if required fields are missing. + """ if not values.get("uri"): raise ValueError("config MILVUS_URI is required") if not values.get("user"): @@ -39,6 +49,9 @@ def validate_config(cls, values: dict) -> dict: return values def to_milvus_params(self): + """ + Convert the configuration to a dictionary of Milvus connection parameters. + """ return { "uri": self.uri, "token": self.token, @@ -49,26 +62,57 @@ def to_milvus_params(self): class MilvusVector(BaseVector): + """ + Milvus vector storage implementation. + """ + def __init__(self, collection_name: str, config: MilvusConfig): super().__init__(collection_name) self._client_config = config self._client = self._init_client(config) - self._consistency_level = "Session" - self._fields: list[str] = [] + self._consistency_level = "Session" # Consistency level for Milvus operations + self._fields: list[str] = [] # List of fields in the collection + self._hybrid_search_enabled = self._check_hybrid_search_support() # Check if hybrid search is supported + + def _check_hybrid_search_support(self) -> bool: + """ + Check if the current Milvus version supports hybrid search. + Returns True if the version is >= 2.5.0, otherwise False. + """ + if not self._client_config.enable_hybrid_search: + return False + + try: + milvus_version = self._client.get_server_version() + return version.parse(milvus_version).base_version >= version.parse("2.5.0").base_version + except Exception as e: + logger.warning(f"Failed to check Milvus version: {str(e)}. Disabling hybrid search.") + return False def get_type(self) -> str: + """ + Get the type of vector storage (Milvus). + """ return VectorType.MILVUS def create(self, texts: list[Document], embeddings: list[list[float]], **kwargs): + """ + Create a collection and add texts with embeddings. + """ index_params = {"metric_type": "IP", "index_type": "HNSW", "params": {"M": 8, "efConstruction": 64}} metadatas = [d.metadata if d.metadata is not None else {} for d in texts] self.create_collection(embeddings, metadatas, index_params) self.add_texts(texts, embeddings) def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs): + """ + Add texts and their embeddings to the collection. + """ insert_dict_list = [] for i in range(len(documents)): insert_dict = { + # Do not need to insert the sparse_vector field separately, as the text_bm25_emb + # function will automatically convert the native text into a sparse vector for us. Field.CONTENT_KEY.value: documents[i].page_content, Field.VECTOR.value: embeddings[i], Field.METADATA_KEY.value: documents[i].metadata, @@ -76,12 +120,11 @@ def add_texts(self, documents: list[Document], embeddings: list[list[float]], ** insert_dict_list.append(insert_dict) # Total insert count total_count = len(insert_dict_list) - pks: list[str] = [] for i in range(0, total_count, 1000): - batch_insert_list = insert_dict_list[i : i + 1000] # Insert into the collection. + batch_insert_list = insert_dict_list[i : i + 1000] try: ids = self._client.insert(collection_name=self._collection_name, data=batch_insert_list) pks.extend(ids) @@ -91,6 +134,9 @@ def add_texts(self, documents: list[Document], embeddings: list[list[float]], ** return pks def get_ids_by_metadata_field(self, key: str, value: str): + """ + Get document IDs by metadata field key and value. + """ result = self._client.query( collection_name=self._collection_name, filter=f'metadata["{key}"] == "{value}"', output_fields=["id"] ) @@ -100,12 +146,18 @@ def get_ids_by_metadata_field(self, key: str, value: str): return None def delete_by_metadata_field(self, key: str, value: str): + """ + Delete documents by metadata field key and value. + """ if self._client.has_collection(self._collection_name): ids = self.get_ids_by_metadata_field(key, value) if ids: self._client.delete(collection_name=self._collection_name, pks=ids) def delete_by_ids(self, ids: list[str]) -> None: + """ + Delete documents by their IDs. + """ if self._client.has_collection(self._collection_name): result = self._client.query( collection_name=self._collection_name, filter=f'metadata["doc_id"] in {ids}', output_fields=["id"] @@ -115,10 +167,16 @@ def delete_by_ids(self, ids: list[str]) -> None: self._client.delete(collection_name=self._collection_name, pks=ids) def delete(self) -> None: + """ + Delete the entire collection. + """ if self._client.has_collection(self._collection_name): self._client.drop_collection(self._collection_name, None) def text_exists(self, id: str) -> bool: + """ + Check if a text with the given ID exists in the collection. + """ if not self._client.has_collection(self._collection_name): return False @@ -128,32 +186,80 @@ def text_exists(self, id: str) -> bool: return len(result) > 0 + def field_exists(self, field: str) -> bool: + """ + Check if a field exists in the collection. + """ + return field in self._fields + + def _process_search_results( + self, results: list[Any], output_fields: list[str], score_threshold: float = 0.0 + ) -> list[Document]: + """ + Common method to process search results + + :param results: Search results + :param output_fields: Fields to be output + :param score_threshold: Score threshold for filtering + :return: List of documents + """ + docs = [] + for result in results[0]: + metadata = result["entity"].get(output_fields[1], {}) + metadata["score"] = result["distance"] + + if result["distance"] > score_threshold: + doc = Document(page_content=result["entity"].get(output_fields[0], ""), metadata=metadata) + docs.append(doc) + + return docs + def search_by_vector(self, query_vector: list[float], **kwargs: Any) -> list[Document]: - # Set search parameters. + """ + Search for documents by vector similarity. + """ results = self._client.search( collection_name=self._collection_name, data=[query_vector], + anns_field=Field.VECTOR.value, limit=kwargs.get("top_k", 4), output_fields=[Field.CONTENT_KEY.value, Field.METADATA_KEY.value], ) - # Organize results. - docs = [] - for result in results[0]: - metadata = result["entity"].get(Field.METADATA_KEY.value) - metadata["score"] = result["distance"] - score_threshold = float(kwargs.get("score_threshold") or 0.0) - if result["distance"] > score_threshold: - doc = Document(page_content=result["entity"].get(Field.CONTENT_KEY.value), metadata=metadata) - docs.append(doc) - return docs + + return self._process_search_results( + results, + output_fields=[Field.CONTENT_KEY.value, Field.METADATA_KEY.value], + score_threshold=float(kwargs.get("score_threshold") or 0.0), + ) def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]: - # milvus/zilliz doesn't support bm25 search - return [] + """ + Search for documents by full-text search (if hybrid search is enabled). + """ + if not self._hybrid_search_enabled or not self.field_exists(Field.SPARSE_VECTOR.value): + logger.warning("Full-text search is not supported in current Milvus version (requires >= 2.5.0)") + return [] + + results = self._client.search( + collection_name=self._collection_name, + data=[query], + anns_field=Field.SPARSE_VECTOR.value, + limit=kwargs.get("top_k", 4), + output_fields=[Field.CONTENT_KEY.value, Field.METADATA_KEY.value], + ) + + return self._process_search_results( + results, + output_fields=[Field.CONTENT_KEY.value, Field.METADATA_KEY.value], + score_threshold=float(kwargs.get("score_threshold") or 0.0), + ) def create_collection( self, embeddings: list, metadatas: Optional[list[dict]] = None, index_params: Optional[dict] = None ): + """ + Create a new collection in Milvus with the specified schema and index parameters. + """ lock_name = "vector_indexing_lock_{}".format(self._collection_name) with redis_client.lock(lock_name, timeout=20): collection_exist_cache_key = "vector_indexing_{}".format(self._collection_name) @@ -161,7 +267,7 @@ def create_collection( return # Grab the existing collection if it exists if not self._client.has_collection(self._collection_name): - from pymilvus import CollectionSchema, DataType, FieldSchema # type: ignore + from pymilvus import CollectionSchema, DataType, FieldSchema, Function, FunctionType # type: ignore from pymilvus.orm.types import infer_dtype_bydata # type: ignore # Determine embedding dim @@ -170,16 +276,36 @@ def create_collection( if metadatas: fields.append(FieldSchema(Field.METADATA_KEY.value, DataType.JSON, max_length=65_535)) - # Create the text field - fields.append(FieldSchema(Field.CONTENT_KEY.value, DataType.VARCHAR, max_length=65_535)) + # Create the text field, enable_analyzer will be set True to support milvus automatically + # transfer text to sparse_vector, reference: https://milvus.io/docs/full-text-search.md + fields.append( + FieldSchema( + Field.CONTENT_KEY.value, + DataType.VARCHAR, + max_length=65_535, + enable_analyzer=self._hybrid_search_enabled, + ) + ) # Create the primary key field fields.append(FieldSchema(Field.PRIMARY_KEY.value, DataType.INT64, is_primary=True, auto_id=True)) # Create the vector field, supports binary or float vectors fields.append(FieldSchema(Field.VECTOR.value, infer_dtype_bydata(embeddings[0]), dim=dim)) + # Create Sparse Vector Index for the collection + if self._hybrid_search_enabled: + fields.append(FieldSchema(Field.SPARSE_VECTOR.value, DataType.SPARSE_FLOAT_VECTOR)) - # Create the schema for the collection schema = CollectionSchema(fields) + # Create custom function to support text to sparse vector by BM25 + if self._hybrid_search_enabled: + bm25_function = Function( + name="text_bm25_emb", + input_field_names=[Field.CONTENT_KEY.value], + output_field_names=[Field.SPARSE_VECTOR.value], + function_type=FunctionType.BM25, + ) + schema.add_function(bm25_function) + for x in schema.fields: self._fields.append(x.name) # Since primary field is auto-id, no need to track it @@ -189,10 +315,15 @@ def create_collection( index_params_obj = IndexParams() index_params_obj.add_index(field_name=Field.VECTOR.value, **index_params) + # Create Sparse Vector Index for the collection + if self._hybrid_search_enabled: + index_params_obj.add_index( + field_name=Field.SPARSE_VECTOR.value, index_type="AUTOINDEX", metric_type="BM25" + ) + # Create the collection - collection_name = self._collection_name self._client.create_collection( - collection_name=collection_name, + collection_name=self._collection_name, schema=schema, index_params=index_params_obj, consistency_level=self._consistency_level, @@ -200,12 +331,22 @@ def create_collection( redis_client.set(collection_exist_cache_key, 1, ex=3600) def _init_client(self, config) -> MilvusClient: + """ + Initialize and return a Milvus client. + """ client = MilvusClient(uri=config.uri, user=config.user, password=config.password, db_name=config.database) return client class MilvusVectorFactory(AbstractVectorFactory): + """ + Factory class for creating MilvusVector instances. + """ + def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings) -> MilvusVector: + """ + Initialize a MilvusVector instance for the given dataset. + """ if dataset.index_struct_dict: class_prefix: str = dataset.index_struct_dict["vector_store"]["class_prefix"] collection_name = class_prefix @@ -222,5 +363,6 @@ def init_vector(self, dataset: Dataset, attributes: list, embeddings: Embeddings user=dify_config.MILVUS_USER or "", password=dify_config.MILVUS_PASSWORD or "", database=dify_config.MILVUS_DATABASE or "", + enable_hybrid_search=dify_config.MILVUS_ENABLE_HYBRID_SEARCH or False, ), ) diff --git a/api/poetry.lock b/api/poetry.lock index fa5b1ebbdede7b..fe80545e7ca682 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -469,13 +469,13 @@ vertex = ["google-auth (>=2,<3)"] [[package]] name = "anyio" -version = "4.7.0" +version = "4.8.0" description = "High level compatibility layer for multiple asynchronous event loop implementations" optional = false python-versions = ">=3.9" files = [ - {file = "anyio-4.7.0-py3-none-any.whl", hash = "sha256:ea60c3723ab42ba6fff7e8ccb0488c898ec538ff4df1f1d5e642c3601d07e352"}, - {file = "anyio-4.7.0.tar.gz", hash = "sha256:2f834749c602966b7d456a7567cafcb309f96482b5081d14ac93ccd457f9dd48"}, + {file = "anyio-4.8.0-py3-none-any.whl", hash = "sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a"}, + {file = "anyio-4.8.0.tar.gz", hash = "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a"}, ] [package.dependencies] @@ -485,7 +485,7 @@ typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} [package.extras] doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] -test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "pytest-mock (>=3.6.1)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] +test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"] trio = ["trio (>=0.26.1)"] [[package]] @@ -856,13 +856,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.35.90" +version = "1.35.94" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.35.90-py3-none-any.whl", hash = "sha256:51dcbe1b32e2ac43dac17091f401a00ce5939f76afe999081802009cce1e92e4"}, - {file = "botocore-1.35.90.tar.gz", hash = "sha256:f007f58e8e3c1ad0412a6ddfae40ed92a7bca571c068cb959902bcf107f2ae48"}, + {file = "botocore-1.35.94-py3-none-any.whl", hash = "sha256:d784d944865d8279c79d2301fc09ac28b5221d4e7328fb4e23c642c253b9932c"}, + {file = "botocore-1.35.94.tar.gz", hash = "sha256:2b3309b356541faa4d88bb957dcac1d8004aa44953c0b7d4521a6cc5d3d5d6ba"}, ] [package.dependencies] @@ -1966,6 +1966,7 @@ files = [ {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:761817a3377ef15ac23cd7834715081791d4ec77f9297ee694ca1ee9c2c7e5eb"}, {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:3c672a53c0fb4725a29c303be906d3c1fa99c32f58abe008a82705f9ee96f40b"}, {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4ac4c9f37eba52cb6fbeaf5b59c152ea976726b865bd4cf87883a7e7006cc543"}, + {file = "cryptography-44.0.0-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:60eb32934076fa07e4316b7b2742fa52cbb190b42c2df2863dbc4230a0a9b385"}, {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ed3534eb1090483c96178fcb0f8893719d96d5274dfde98aa6add34614e97c8e"}, {file = "cryptography-44.0.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:f3f6fdfa89ee2d9d496e2c087cebef9d4fcbb0ad63c40e821b39f74bf48d9c5e"}, {file = "cryptography-44.0.0-cp37-abi3-win32.whl", hash = "sha256:eb33480f1bad5b78233b0ad3e1b0be21e8ef1da745d8d2aecbb20671658b9053"}, @@ -1976,6 +1977,7 @@ files = [ {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:c5eb858beed7835e5ad1faba59e865109f3e52b3783b9ac21e7e47dc5554e289"}, {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f53c2c87e0fb4b0c00fa9571082a057e37690a8f12233306161c8f4b819960b7"}, {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:9e6fc8a08e116fb7c7dd1f040074c9d7b51d74a8ea40d4df2fc7aa08b76b9e6c"}, + {file = "cryptography-44.0.0-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9abcc2e083cbe8dde89124a47e5e53ec38751f0d7dfd36801008f316a127d7ba"}, {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:d2436114e46b36d00f8b72ff57e598978b37399d2786fd39793c36c6d5cb1c64"}, {file = "cryptography-44.0.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a01956ddfa0a6790d594f5b34fc1bfa6098aca434696a03cfdbe469b8ed79285"}, {file = "cryptography-44.0.0-cp39-abi3-win32.whl", hash = "sha256:eca27345e1214d1b9f9490d200f9db5a874479be914199194e746c893788d417"}, @@ -2322,13 +2324,13 @@ files = [ [[package]] name = "elastic-transport" -version = "8.15.1" +version = "8.17.0" description = "Transport classes and utilities shared among Python Elastic client libraries" optional = false python-versions = ">=3.8" files = [ - {file = "elastic_transport-8.15.1-py3-none-any.whl", hash = "sha256:b5e82ff1679d8c7705a03fd85c7f6ef85d6689721762d41228dd312e34f331fc"}, - {file = "elastic_transport-8.15.1.tar.gz", hash = "sha256:9cac4ab5cf9402668cf305ae0b7d93ddc0c7b61461d6d1027850db6da9cc5742"}, + {file = "elastic_transport-8.17.0-py3-none-any.whl", hash = "sha256:59f553300866750e67a38828fede000576562a0e66930c641adb75249e0c95af"}, + {file = "elastic_transport-8.17.0.tar.gz", hash = "sha256:e755f38f99fa6ec5456e236b8e58f0eb18873ac8fe710f74b91a16dd562de2a5"}, ] [package.dependencies] @@ -2372,27 +2374,6 @@ files = [ [package.extras] dev = ["coverage", "pytest (>=7.4.4)"] -[[package]] -name = "environs" -version = "9.5.0" -description = "simplified environment variable parsing" -optional = false -python-versions = ">=3.6" -files = [ - {file = "environs-9.5.0-py2.py3-none-any.whl", hash = "sha256:1e549569a3de49c05f856f40bce86979e7d5ffbbc4398e7f338574c220189124"}, - {file = "environs-9.5.0.tar.gz", hash = "sha256:a76307b36fbe856bdca7ee9161e6c466fd7fcffc297109a118c59b54e27e30c9"}, -] - -[package.dependencies] -marshmallow = ">=3.0.0" -python-dotenv = "*" - -[package.extras] -dev = ["dj-database-url", "dj-email-url", "django-cache-url", "flake8 (==4.0.1)", "flake8-bugbear (==21.9.2)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)", "pytest", "tox"] -django = ["dj-database-url", "dj-email-url", "django-cache-url"] -lint = ["flake8 (==4.0.1)", "flake8-bugbear (==21.9.2)", "mypy (==0.910)", "pre-commit (>=2.4,<3.0)"] -tests = ["dj-database-url", "dj-email-url", "django-cache-url", "pytest"] - [[package]] name = "esdk-obs-python" version = "3.24.6.1" @@ -3657,70 +3638,70 @@ protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4 [[package]] name = "grpcio" -version = "1.68.1" +version = "1.67.1" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.8" files = [ - {file = "grpcio-1.68.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:d35740e3f45f60f3c37b1e6f2f4702c23867b9ce21c6410254c9c682237da68d"}, - {file = "grpcio-1.68.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:d99abcd61760ebb34bdff37e5a3ba333c5cc09feda8c1ad42547bea0416ada78"}, - {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:f8261fa2a5f679abeb2a0a93ad056d765cdca1c47745eda3f2d87f874ff4b8c9"}, - {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0feb02205a27caca128627bd1df4ee7212db051019a9afa76f4bb6a1a80ca95e"}, - {file = "grpcio-1.68.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:919d7f18f63bcad3a0f81146188e90274fde800a94e35d42ffe9eadf6a9a6330"}, - {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:963cc8d7d79b12c56008aabd8b457f400952dbea8997dd185f155e2f228db079"}, - {file = "grpcio-1.68.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ccf2ebd2de2d6661e2520dae293298a3803a98ebfc099275f113ce1f6c2a80f1"}, - {file = "grpcio-1.68.1-cp310-cp310-win32.whl", hash = "sha256:2cc1fd04af8399971bcd4f43bd98c22d01029ea2e56e69c34daf2bf8470e47f5"}, - {file = "grpcio-1.68.1-cp310-cp310-win_amd64.whl", hash = "sha256:ee2e743e51cb964b4975de572aa8fb95b633f496f9fcb5e257893df3be854746"}, - {file = "grpcio-1.68.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:55857c71641064f01ff0541a1776bfe04a59db5558e82897d35a7793e525774c"}, - {file = "grpcio-1.68.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4b177f5547f1b995826ef529d2eef89cca2f830dd8b2c99ffd5fde4da734ba73"}, - {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:3522c77d7e6606d6665ec8d50e867f13f946a4e00c7df46768f1c85089eae515"}, - {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9d1fae6bbf0816415b81db1e82fb3bf56f7857273c84dcbe68cbe046e58e1ccd"}, - {file = "grpcio-1.68.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:298ee7f80e26f9483f0b6f94cc0a046caf54400a11b644713bb5b3d8eb387600"}, - {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cbb5780e2e740b6b4f2d208e90453591036ff80c02cc605fea1af8e6fc6b1bbe"}, - {file = "grpcio-1.68.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ddda1aa22495d8acd9dfbafff2866438d12faec4d024ebc2e656784d96328ad0"}, - {file = "grpcio-1.68.1-cp311-cp311-win32.whl", hash = "sha256:b33bd114fa5a83f03ec6b7b262ef9f5cac549d4126f1dc702078767b10c46ed9"}, - {file = "grpcio-1.68.1-cp311-cp311-win_amd64.whl", hash = "sha256:7f20ebec257af55694d8f993e162ddf0d36bd82d4e57f74b31c67b3c6d63d8b2"}, - {file = "grpcio-1.68.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:8829924fffb25386995a31998ccbbeaa7367223e647e0122043dfc485a87c666"}, - {file = "grpcio-1.68.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:3aed6544e4d523cd6b3119b0916cef3d15ef2da51e088211e4d1eb91a6c7f4f1"}, - {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:4efac5481c696d5cb124ff1c119a78bddbfdd13fc499e3bc0ca81e95fc573684"}, - {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ab2d912ca39c51f46baf2a0d92aa265aa96b2443266fc50d234fa88bf877d8e"}, - {file = "grpcio-1.68.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c87ce2a97434dffe7327a4071839ab8e8bffd0054cc74cbe971fba98aedd60"}, - {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e4842e4872ae4ae0f5497bf60a0498fa778c192cc7a9e87877abd2814aca9475"}, - {file = "grpcio-1.68.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:255b1635b0ed81e9f91da4fcc8d43b7ea5520090b9a9ad9340d147066d1d3613"}, - {file = "grpcio-1.68.1-cp312-cp312-win32.whl", hash = "sha256:7dfc914cc31c906297b30463dde0b9be48e36939575eaf2a0a22a8096e69afe5"}, - {file = "grpcio-1.68.1-cp312-cp312-win_amd64.whl", hash = "sha256:a0c8ddabef9c8f41617f213e527254c41e8b96ea9d387c632af878d05db9229c"}, - {file = "grpcio-1.68.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:a47faedc9ea2e7a3b6569795c040aae5895a19dde0c728a48d3c5d7995fda385"}, - {file = "grpcio-1.68.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:390eee4225a661c5cd133c09f5da1ee3c84498dc265fd292a6912b65c421c78c"}, - {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:66a24f3d45c33550703f0abb8b656515b0ab777970fa275693a2f6dc8e35f1c1"}, - {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c08079b4934b0bf0a8847f42c197b1d12cba6495a3d43febd7e99ecd1cdc8d54"}, - {file = "grpcio-1.68.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8720c25cd9ac25dd04ee02b69256d0ce35bf8a0f29e20577427355272230965a"}, - {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:04cfd68bf4f38f5bb959ee2361a7546916bd9a50f78617a346b3aeb2b42e2161"}, - {file = "grpcio-1.68.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c28848761a6520c5c6071d2904a18d339a796ebe6b800adc8b3f474c5ce3c3ad"}, - {file = "grpcio-1.68.1-cp313-cp313-win32.whl", hash = "sha256:77d65165fc35cff6e954e7fd4229e05ec76102d4406d4576528d3a3635fc6172"}, - {file = "grpcio-1.68.1-cp313-cp313-win_amd64.whl", hash = "sha256:a8040f85dcb9830d8bbb033ae66d272614cec6faceee88d37a88a9bd1a7a704e"}, - {file = "grpcio-1.68.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:eeb38ff04ab6e5756a2aef6ad8d94e89bb4a51ef96e20f45c44ba190fa0bcaad"}, - {file = "grpcio-1.68.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:8a3869a6661ec8f81d93f4597da50336718bde9eb13267a699ac7e0a1d6d0bea"}, - {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:2c4cec6177bf325eb6faa6bd834d2ff6aa8bb3b29012cceb4937b86f8b74323c"}, - {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12941d533f3cd45d46f202e3667be8ebf6bcb3573629c7ec12c3e211d99cfccf"}, - {file = "grpcio-1.68.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80af6f1e69c5e68a2be529990684abdd31ed6622e988bf18850075c81bb1ad6e"}, - {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:e8dbe3e00771bfe3d04feed8210fc6617006d06d9a2679b74605b9fed3e8362c"}, - {file = "grpcio-1.68.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:83bbf5807dc3ee94ce1de2dfe8a356e1d74101e4b9d7aa8c720cc4818a34aded"}, - {file = "grpcio-1.68.1-cp38-cp38-win32.whl", hash = "sha256:8cb620037a2fd9eeee97b4531880e439ebfcd6d7d78f2e7dcc3726428ab5ef63"}, - {file = "grpcio-1.68.1-cp38-cp38-win_amd64.whl", hash = "sha256:52fbf85aa71263380d330f4fce9f013c0798242e31ede05fcee7fbe40ccfc20d"}, - {file = "grpcio-1.68.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:cb400138e73969eb5e0535d1d06cae6a6f7a15f2cc74add320e2130b8179211a"}, - {file = "grpcio-1.68.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a1b988b40f2fd9de5c820f3a701a43339d8dcf2cb2f1ca137e2c02671cc83ac1"}, - {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:96f473cdacfdd506008a5d7579c9f6a7ff245a9ade92c3c0265eb76cc591914f"}, - {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:37ea3be171f3cf3e7b7e412a98b77685eba9d4fd67421f4a34686a63a65d99f9"}, - {file = "grpcio-1.68.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ceb56c4285754e33bb3c2fa777d055e96e6932351a3082ce3559be47f8024f0"}, - {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:dffd29a2961f3263a16d73945b57cd44a8fd0b235740cb14056f0612329b345e"}, - {file = "grpcio-1.68.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:025f790c056815b3bf53da850dd70ebb849fd755a4b1ac822cb65cd631e37d43"}, - {file = "grpcio-1.68.1-cp39-cp39-win32.whl", hash = "sha256:1098f03dedc3b9810810568060dea4ac0822b4062f537b0f53aa015269be0a76"}, - {file = "grpcio-1.68.1-cp39-cp39-win_amd64.whl", hash = "sha256:334ab917792904245a028f10e803fcd5b6f36a7b2173a820c0b5b076555825e1"}, - {file = "grpcio-1.68.1.tar.gz", hash = "sha256:44a8502dd5de653ae6a73e2de50a401d84184f0331d0ac3daeb044e66d5c5054"}, -] - -[package.extras] -protobuf = ["grpcio-tools (>=1.68.1)"] + {file = "grpcio-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:8b0341d66a57f8a3119b77ab32207072be60c9bf79760fa609c5609f2deb1f3f"}, + {file = "grpcio-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:f5a27dddefe0e2357d3e617b9079b4bfdc91341a91565111a21ed6ebbc51b22d"}, + {file = "grpcio-1.67.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:43112046864317498a33bdc4797ae6a268c36345a910de9b9c17159d8346602f"}, + {file = "grpcio-1.67.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9b929f13677b10f63124c1a410994a401cdd85214ad83ab67cc077fc7e480f0"}, + {file = "grpcio-1.67.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7d1797a8a3845437d327145959a2c0c47c05947c9eef5ff1a4c80e499dcc6fa"}, + {file = "grpcio-1.67.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0489063974d1452436139501bf6b180f63d4977223ee87488fe36858c5725292"}, + {file = "grpcio-1.67.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9fd042de4a82e3e7aca44008ee2fb5da01b3e5adb316348c21980f7f58adc311"}, + {file = "grpcio-1.67.1-cp310-cp310-win32.whl", hash = "sha256:638354e698fd0c6c76b04540a850bf1db27b4d2515a19fcd5cf645c48d3eb1ed"}, + {file = "grpcio-1.67.1-cp310-cp310-win_amd64.whl", hash = "sha256:608d87d1bdabf9e2868b12338cd38a79969eaf920c89d698ead08f48de9c0f9e"}, + {file = "grpcio-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:7818c0454027ae3384235a65210bbf5464bd715450e30a3d40385453a85a70cb"}, + {file = "grpcio-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ea33986b70f83844cd00814cee4451055cd8cab36f00ac64a31f5bb09b31919e"}, + {file = "grpcio-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c7a01337407dd89005527623a4a72c5c8e2894d22bead0895306b23c6695698f"}, + {file = "grpcio-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80b866f73224b0634f4312a4674c1be21b2b4afa73cb20953cbbb73a6b36c3cc"}, + {file = "grpcio-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fff78ba10d4250bfc07a01bd6254a6d87dc67f9627adece85c0b2ed754fa96"}, + {file = "grpcio-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8a23cbcc5bb11ea7dc6163078be36c065db68d915c24f5faa4f872c573bb400f"}, + {file = "grpcio-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1a65b503d008f066e994f34f456e0647e5ceb34cfcec5ad180b1b44020ad4970"}, + {file = "grpcio-1.67.1-cp311-cp311-win32.whl", hash = "sha256:e29ca27bec8e163dca0c98084040edec3bc49afd10f18b412f483cc68c712744"}, + {file = "grpcio-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:786a5b18544622bfb1e25cc08402bd44ea83edfb04b93798d85dca4d1a0b5be5"}, + {file = "grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953"}, + {file = "grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb"}, + {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0"}, + {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af"}, + {file = "grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e"}, + {file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75"}, + {file = "grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38"}, + {file = "grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78"}, + {file = "grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc"}, + {file = "grpcio-1.67.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:aa0162e56fd10a5547fac8774c4899fc3e18c1aa4a4759d0ce2cd00d3696ea6b"}, + {file = "grpcio-1.67.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:beee96c8c0b1a75d556fe57b92b58b4347c77a65781ee2ac749d550f2a365dc1"}, + {file = "grpcio-1.67.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:a93deda571a1bf94ec1f6fcda2872dad3ae538700d94dc283c672a3b508ba3af"}, + {file = "grpcio-1.67.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e6f255980afef598a9e64a24efce87b625e3e3c80a45162d111a461a9f92955"}, + {file = "grpcio-1.67.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e838cad2176ebd5d4a8bb03955138d6589ce9e2ce5d51c3ada34396dbd2dba8"}, + {file = "grpcio-1.67.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:a6703916c43b1d468d0756c8077b12017a9fcb6a1ef13faf49e67d20d7ebda62"}, + {file = "grpcio-1.67.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:917e8d8994eed1d86b907ba2a61b9f0aef27a2155bca6cbb322430fc7135b7bb"}, + {file = "grpcio-1.67.1-cp313-cp313-win32.whl", hash = "sha256:e279330bef1744040db8fc432becc8a727b84f456ab62b744d3fdb83f327e121"}, + {file = "grpcio-1.67.1-cp313-cp313-win_amd64.whl", hash = "sha256:fa0c739ad8b1996bd24823950e3cb5152ae91fca1c09cc791190bf1627ffefba"}, + {file = "grpcio-1.67.1-cp38-cp38-linux_armv7l.whl", hash = "sha256:178f5db771c4f9a9facb2ab37a434c46cb9be1a75e820f187ee3d1e7805c4f65"}, + {file = "grpcio-1.67.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0f3e49c738396e93b7ba9016e153eb09e0778e776df6090c1b8c91877cc1c426"}, + {file = "grpcio-1.67.1-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:24e8a26dbfc5274d7474c27759b54486b8de23c709d76695237515bc8b5baeab"}, + {file = "grpcio-1.67.1-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3b6c16489326d79ead41689c4b84bc40d522c9a7617219f4ad94bc7f448c5085"}, + {file = "grpcio-1.67.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e6a4dcf5af7bbc36fd9f81c9f372e8ae580870a9e4b6eafe948cd334b81cf3"}, + {file = "grpcio-1.67.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:95b5f2b857856ed78d72da93cd7d09b6db8ef30102e5e7fe0961fe4d9f7d48e8"}, + {file = "grpcio-1.67.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b49359977c6ec9f5d0573ea4e0071ad278ef905aa74e420acc73fd28ce39e9ce"}, + {file = "grpcio-1.67.1-cp38-cp38-win32.whl", hash = "sha256:f5b76ff64aaac53fede0cc93abf57894ab2a7362986ba22243d06218b93efe46"}, + {file = "grpcio-1.67.1-cp38-cp38-win_amd64.whl", hash = "sha256:804c6457c3cd3ec04fe6006c739579b8d35c86ae3298ffca8de57b493524b771"}, + {file = "grpcio-1.67.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:a25bdea92b13ff4d7790962190bf6bf5c4639876e01c0f3dda70fc2769616335"}, + {file = "grpcio-1.67.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:cdc491ae35a13535fd9196acb5afe1af37c8237df2e54427be3eecda3653127e"}, + {file = "grpcio-1.67.1-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:85f862069b86a305497e74d0dc43c02de3d1d184fc2c180993aa8aa86fbd19b8"}, + {file = "grpcio-1.67.1-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ec74ef02010186185de82cc594058a3ccd8d86821842bbac9873fd4a2cf8be8d"}, + {file = "grpcio-1.67.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:01f616a964e540638af5130469451cf580ba8c7329f45ca998ab66e0c7dcdb04"}, + {file = "grpcio-1.67.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:299b3d8c4f790c6bcca485f9963b4846dd92cf6f1b65d3697145d005c80f9fe8"}, + {file = "grpcio-1.67.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:60336bff760fbb47d7e86165408126f1dded184448e9a4c892189eb7c9d3f90f"}, + {file = "grpcio-1.67.1-cp39-cp39-win32.whl", hash = "sha256:5ed601c4c6008429e3d247ddb367fe8c7259c355757448d7c1ef7bd4a6739e8e"}, + {file = "grpcio-1.67.1-cp39-cp39-win_amd64.whl", hash = "sha256:5db70d32d6703b89912af16d6d45d78406374a8b8ef0d28140351dd0ec610e98"}, + {file = "grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732"}, +] + +[package.extras] +protobuf = ["grpcio-tools (>=1.67.1)"] [[package]] name = "grpcio-status" @@ -4217,13 +4198,13 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs [[package]] name = "importlib-resources" -version = "6.4.5" +version = "6.5.2" description = "Read resources from Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"}, - {file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"}, + {file = "importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec"}, + {file = "importlib_resources-6.5.2.tar.gz", hash = "sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c"}, ] [package.extras] @@ -5111,13 +5092,13 @@ files = [ [[package]] name = "marshmallow" -version = "3.23.2" +version = "3.24.1" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.9" files = [ - {file = "marshmallow-3.23.2-py3-none-any.whl", hash = "sha256:bcaf2d6fd74fb1459f8450e85d994997ad3e70036452cbfa4ab685acb19479b3"}, - {file = "marshmallow-3.23.2.tar.gz", hash = "sha256:c448ac6455ca4d794773f00bae22c2f351d62d739929f761dce5eacb5c468d7f"}, + {file = "marshmallow-3.24.1-py3-none-any.whl", hash = "sha256:ddb5c9987017d37be351c184e4e867e7bf55f7331f4da730dedad6b7af662cdd"}, + {file = "marshmallow-3.24.1.tar.gz", hash = "sha256:efdcb656ac8788f0e3d1d938f8dc0f237bf1a99aff8f6dfbffa594981641cea0"}, ] [package.dependencies] @@ -5125,7 +5106,7 @@ packaging = ">=17.0" [package.extras] dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"] -docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.14)", "sphinx (==8.1.3)", "sphinx-issues (==5.0.0)", "sphinx-version-warning (==1.1.2)"] +docs = ["alabaster (==1.0.0)", "autodocsumm (==0.2.14)", "sphinx (==8.1.3)", "sphinx-issues (==5.0.0)"] tests = ["pytest", "simplejson"] [[package]] @@ -5646,6 +5627,17 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "ndjson" +version = "0.3.1" +description = "JsonDecoder for ndjson" +optional = false +python-versions = "*" +files = [ + {file = "ndjson-0.3.1-py2.py3-none-any.whl", hash = "sha256:839c22275e6baa3040077b83c005ac24199b94973309a8a1809be962c753a410"}, + {file = "ndjson-0.3.1.tar.gz", hash = "sha256:bf9746cb6bb1cb53d172cda7f154c07c786d665ff28341e4e689b796b229e5d6"}, +] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -6027,13 +6019,13 @@ files = [ [[package]] name = "opencensus-ext-azure" -version = "1.1.13" +version = "1.1.14" description = "OpenCensus Azure Monitor Exporter" optional = false python-versions = "*" files = [ - {file = "opencensus-ext-azure-1.1.13.tar.gz", hash = "sha256:aec30472177005379ba56a702a097d618c5f57558e1bb6676ec75f948130692a"}, - {file = "opencensus_ext_azure-1.1.13-py2.py3-none-any.whl", hash = "sha256:06001fac6f8588ba00726a3a7c6c7f2fc88bc8ad12a65afdca657923085393dd"}, + {file = "opencensus-ext-azure-1.1.14.tar.gz", hash = "sha256:c9c6ebad542aeb61813322e627d5889a563e7b8c4e024bf58469d06db73ab148"}, + {file = "opencensus_ext_azure-1.1.14-py2.py3-none-any.whl", hash = "sha256:a1f6870d6e4e312832e6ebd95df28ed499ac637c36cbd77665fe06e24ddeb2f1"}, ] [package.dependencies] @@ -6793,13 +6785,13 @@ pydantic = ">=1.9,<3.0" [[package]] name = "posthog" -version = "3.7.4" +version = "3.7.5" description = "Integrate PostHog into any python application." optional = false python-versions = "*" files = [ - {file = "posthog-3.7.4-py2.py3-none-any.whl", hash = "sha256:21c18c6bf43b2de303ea4cd6e95804cc0f24c20cb2a96a8fd09da2ed50b62faa"}, - {file = "posthog-3.7.4.tar.gz", hash = "sha256:19384bd09d330f9787a7e2446aba14c8057ece56144970ea2791072d4e40cd36"}, + {file = "posthog-3.7.5-py2.py3-none-any.whl", hash = "sha256:022132c17069dde03c5c5904e2ae1b9bd68d5059cbc5a8dffc5c1537a1b71cb5"}, + {file = "posthog-3.7.5.tar.gz", hash = "sha256:8ba40ab623da35db72715fc87fe7dccb7fc272ced92581fe31db2d4dbe7ad761"}, ] [package.dependencies] @@ -6841,20 +6833,20 @@ dill = ["dill (>=0.3.9)"] [[package]] name = "primp" -version = "0.9.2" +version = "0.10.0" description = "HTTP client that can impersonate web browsers, mimicking their headers and `TLS/JA3/JA4/HTTP2` fingerprints" optional = false python-versions = ">=3.8" files = [ - {file = "primp-0.9.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:a3179640e633be843ed5daba5c4e3086ad91f77c7bb40a9db06326f28d56b12b"}, - {file = "primp-0.9.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:94a5da8ba25f74152b43bc16a7591dfb5d7d30a5827dc0a0f96a956f7d3616be"}, - {file = "primp-0.9.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0668c0abb6d56fc8b0a918179b1d0f68e7267c1dc632e2b683c618317e13143f"}, - {file = "primp-0.9.2-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:a9c29a4b8eabfc28a1746d2fe93d33b9fcf2e81e642dd0e3eaecede60cc36b7d"}, - {file = "primp-0.9.2-cp38-abi3-manylinux_2_34_armv7l.whl", hash = "sha256:04d499308a101b06b40f5fda1bdc795db5731cd0dfbb1a8873f4acd07c085b1d"}, - {file = "primp-0.9.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:4cd5daf39034a0a8c96cdc0c4c306184c6f2b1b2a0b39dc3294d79ed28a6f7fe"}, - {file = "primp-0.9.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:8d14653434837eb431b3cf7ca006647d7a196906e48bba96bb600ba2ba70bcdc"}, - {file = "primp-0.9.2-cp38-abi3-win_amd64.whl", hash = "sha256:80d9f07564dc9b25b1a9676df770561418557c124fedecae84f6491a1974b61d"}, - {file = "primp-0.9.2.tar.gz", hash = "sha256:5b95666c25b9107eab3c05a89cb7b1748d5122e57c57b25bfc3249d525c45300"}, + {file = "primp-0.10.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:7a91a089bf2962b5b56c8d83d09535eb81cf55b53c09d83208b9e5a715cf2c17"}, + {file = "primp-0.10.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:0128453cce81552f7aa6ac2bf9b8741b7816cdb2d10536e62c77daaf6483b9af"}, + {file = "primp-0.10.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a959e9a83cff0ae7a85a02cc183e4db636f69ff41dddb7c4e32f997924923417"}, + {file = "primp-0.10.0-cp38-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:8e711cfa019fa9bdc0cba4d5d596f319c884a4329e505bd73e92eee0b024061a"}, + {file = "primp-0.10.0-cp38-abi3-manylinux_2_34_armv7l.whl", hash = "sha256:b859336d9a35669b68a29c5d8f050e0dca380452dabf6c9667bb8599f010d164"}, + {file = "primp-0.10.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:dc875cc9a733fe3e6344a37f2b5888e0a9605bb37807fc3009f3b03786408f34"}, + {file = "primp-0.10.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:a27c5d997c37bf8237963c11e376eaa66e7eccee39164e3e259a1c3767c304d6"}, + {file = "primp-0.10.0-cp38-abi3-win_amd64.whl", hash = "sha256:7fe94c3164c2efffff08f7f54c018ac445112961b3ce4f4f499315ba0a9d1ef3"}, + {file = "primp-0.10.0.tar.gz", hash = "sha256:93142590a5a1958240ee5b74faaf2f55185ed499ccaabc622d71cb0cc8a47a0b"}, ] [package.extras] @@ -7464,13 +7456,13 @@ files = [ [[package]] name = "pygments" -version = "2.18.0" +version = "2.19.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" files = [ - {file = "pygments-2.18.0-py3-none-any.whl", hash = "sha256:b8e6aca0523f3ab76fee51799c488e38782ac06eafcf95e7ba832985c8e7b13a"}, - {file = "pygments-2.18.0.tar.gz", hash = "sha256:786ff802f32e91311bff3889f6e9a86e81505fe99f2735bb6d60ae0c5004f199"}, + {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, + {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, ] [package.extras] @@ -7498,21 +7490,21 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] [[package]] name = "pymilvus" -version = "2.4.9" +version = "2.5.3" description = "Python Sdk for Milvus" optional = false python-versions = ">=3.8" files = [ - {file = "pymilvus-2.4.9-py3-none-any.whl", hash = "sha256:45313607d2c164064bdc44e0f933cb6d6afa92e9efcc7f357c5240c57db58fbe"}, - {file = "pymilvus-2.4.9.tar.gz", hash = "sha256:0937663700007c23a84cfc0656160b301f6ff9247aaec4c96d599a6b43572136"}, + {file = "pymilvus-2.5.3-py3-none-any.whl", hash = "sha256:64ca63594284586937274800be27a402f3be2d078130bf81d94ab8d7798ac9c8"}, + {file = "pymilvus-2.5.3.tar.gz", hash = "sha256:68bc3797b7a14c494caf116cee888894ffd6eba7b96a3ac841be85d60694cc5d"}, ] [package.dependencies] -environs = "<=9.5.0" -grpcio = ">=1.49.1" -milvus-lite = {version = ">=2.4.0,<2.5.0", markers = "sys_platform != \"win32\""} +grpcio = ">=1.49.1,<=1.67.1" +milvus-lite = {version = ">=2.4.0", markers = "sys_platform != \"win32\""} pandas = ">=1.2.4" protobuf = ">=3.20.0" +python-dotenv = ">=1.0.1,<2.0.0" setuptools = ">69" ujson = ">=2.0.0" @@ -7916,13 +7908,13 @@ typing-extensions = ">=4.9.0" [[package]] name = "python-dotenv" -version = "1.0.0" +version = "1.0.1" description = "Read key-value pairs from a .env file and set them as environment variables" optional = false python-versions = ">=3.8" files = [ - {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"}, - {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"}, + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, ] [package.extras] @@ -8701,29 +8693,29 @@ pyasn1 = ">=0.1.3" [[package]] name = "ruff" -version = "0.8.5" +version = "0.8.6" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.8.5-py3-none-linux_armv6l.whl", hash = "sha256:5ad11a5e3868a73ca1fa4727fe7e33735ea78b416313f4368c504dbeb69c0f88"}, - {file = "ruff-0.8.5-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:f69ab37771ea7e0715fead8624ec42996d101269a96e31f4d31be6fc33aa19b7"}, - {file = "ruff-0.8.5-py3-none-macosx_11_0_arm64.whl", hash = "sha256:b5462d7804558ccff9c08fe8cbf6c14b7efe67404316696a2dde48297b1925bb"}, - {file = "ruff-0.8.5-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d56de7220a35607f9fe59f8a6d018e14504f7b71d784d980835e20fc0611cd50"}, - {file = "ruff-0.8.5-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9d99cf80b0429cbebf31cbbf6f24f05a29706f0437c40413d950e67e2d4faca4"}, - {file = "ruff-0.8.5-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b75ac29715ac60d554a049dbb0ef3b55259076181c3369d79466cb130eb5afd"}, - {file = "ruff-0.8.5-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:c9d526a62c9eda211b38463528768fd0ada25dad524cb33c0e99fcff1c67b5dc"}, - {file = "ruff-0.8.5-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:587c5e95007612c26509f30acc506c874dab4c4abbacd0357400bd1aa799931b"}, - {file = "ruff-0.8.5-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:622b82bf3429ff0e346835ec213aec0a04d9730480cbffbb6ad9372014e31bbd"}, - {file = "ruff-0.8.5-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f99be814d77a5dac8a8957104bdd8c359e85c86b0ee0e38dca447cb1095f70fb"}, - {file = "ruff-0.8.5-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c01c048f9c3385e0fd7822ad0fd519afb282af9cf1778f3580e540629df89725"}, - {file = "ruff-0.8.5-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7512e8cb038db7f5db6aae0e24735ff9ea03bb0ed6ae2ce534e9baa23c1dc9ea"}, - {file = "ruff-0.8.5-py3-none-musllinux_1_2_i686.whl", hash = "sha256:762f113232acd5b768d6b875d16aad6b00082add40ec91c927f0673a8ec4ede8"}, - {file = "ruff-0.8.5-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:03a90200c5dfff49e4c967b405f27fdfa81594cbb7c5ff5609e42d7fe9680da5"}, - {file = "ruff-0.8.5-py3-none-win32.whl", hash = "sha256:8710ffd57bdaa6690cbf6ecff19884b8629ec2a2a2a2f783aa94b1cc795139ed"}, - {file = "ruff-0.8.5-py3-none-win_amd64.whl", hash = "sha256:4020d8bf8d3a32325c77af452a9976a9ad6455773bcb94991cf15bd66b347e47"}, - {file = "ruff-0.8.5-py3-none-win_arm64.whl", hash = "sha256:134ae019ef13e1b060ab7136e7828a6d83ea727ba123381307eb37c6bd5e01cb"}, - {file = "ruff-0.8.5.tar.gz", hash = "sha256:1098d36f69831f7ff2a1da3e6407d5fbd6dfa2559e4f74ff2d260c5588900317"}, + {file = "ruff-0.8.6-py3-none-linux_armv6l.whl", hash = "sha256:defed167955d42c68b407e8f2e6f56ba52520e790aba4ca707a9c88619e580e3"}, + {file = "ruff-0.8.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:54799ca3d67ae5e0b7a7ac234baa657a9c1784b48ec954a094da7c206e0365b1"}, + {file = "ruff-0.8.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e88b8f6d901477c41559ba540beeb5a671e14cd29ebd5683903572f4b40a9807"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0509e8da430228236a18a677fcdb0c1f102dd26d5520f71f79b094963322ed25"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:91a7ddb221779871cf226100e677b5ea38c2d54e9e2c8ed847450ebbdf99b32d"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:248b1fb3f739d01d528cc50b35ee9c4812aa58cc5935998e776bf8ed5b251e75"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:bc3c083c50390cf69e7e1b5a5a7303898966be973664ec0c4a4acea82c1d4315"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52d587092ab8df308635762386f45f4638badb0866355b2b86760f6d3c076188"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:61323159cf21bc3897674e5adb27cd9e7700bab6b84de40d7be28c3d46dc67cf"}, + {file = "ruff-0.8.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ae4478b1471fc0c44ed52a6fb787e641a2ac58b1c1f91763bafbc2faddc5117"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0c000a471d519b3e6cfc9c6680025d923b4ca140ce3e4612d1a2ef58e11f11fe"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9257aa841e9e8d9b727423086f0fa9a86b6b420fbf4bf9e1465d1250ce8e4d8d"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:45a56f61b24682f6f6709636949ae8cc82ae229d8d773b4c76c09ec83964a95a"}, + {file = "ruff-0.8.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:496dd38a53aa173481a7d8866bcd6451bd934d06976a2505028a50583e001b76"}, + {file = "ruff-0.8.6-py3-none-win32.whl", hash = "sha256:e169ea1b9eae61c99b257dc83b9ee6c76f89042752cb2d83486a7d6e48e8f764"}, + {file = "ruff-0.8.6-py3-none-win_amd64.whl", hash = "sha256:f1d70bef3d16fdc897ee290d7d20da3cbe4e26349f62e8a0274e7a3f4ce7a905"}, + {file = "ruff-0.8.6-py3-none-win_arm64.whl", hash = "sha256:7d7fc2377a04b6e04ffe588caad613d0c460eb2ecba4c0ccbbfe2bc973cbc162"}, + {file = "ruff-0.8.6.tar.gz", hash = "sha256:dcad24b81b62650b0eb8814f576fc65cfee8674772a6e24c9b747911801eeaa5"}, ] [[package]] @@ -9005,53 +8997,60 @@ tests = ["black (>=24.3.0)", "matplotlib (>=3.3.4)", "mypy (>=1.9)", "numpydoc ( [[package]] name = "scipy" -version = "1.14.1" +version = "1.15.0" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.10" files = [ - {file = "scipy-1.14.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:b28d2ca4add7ac16ae8bb6632a3c86e4b9e4d52d3e34267f6e1b0c1f8d87e389"}, - {file = "scipy-1.14.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d0d2821003174de06b69e58cef2316a6622b60ee613121199cb2852a873f8cf3"}, - {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8bddf15838ba768bb5f5083c1ea012d64c9a444e16192762bd858f1e126196d0"}, - {file = "scipy-1.14.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:97c5dddd5932bd2a1a31c927ba5e1463a53b87ca96b5c9bdf5dfd6096e27efc3"}, - {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2ff0a7e01e422c15739ecd64432743cf7aae2b03f3084288f399affcefe5222d"}, - {file = "scipy-1.14.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e32dced201274bf96899e6491d9ba3e9a5f6b336708656466ad0522d8528f69"}, - {file = "scipy-1.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:8426251ad1e4ad903a4514712d2fa8fdd5382c978010d1c6f5f37ef286a713ad"}, - {file = "scipy-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:a49f6ed96f83966f576b33a44257d869756df6cf1ef4934f59dd58b25e0327e5"}, - {file = "scipy-1.14.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:2da0469a4ef0ecd3693761acbdc20f2fdeafb69e6819cc081308cc978153c675"}, - {file = "scipy-1.14.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:c0ee987efa6737242745f347835da2cc5bb9f1b42996a4d97d5c7ff7928cb6f2"}, - {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3a1b111fac6baec1c1d92f27e76511c9e7218f1695d61b59e05e0fe04dc59617"}, - {file = "scipy-1.14.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8475230e55549ab3f207bff11ebfc91c805dc3463ef62eda3ccf593254524ce8"}, - {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:278266012eb69f4a720827bdd2dc54b2271c97d84255b2faaa8f161a158c3b37"}, - {file = "scipy-1.14.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fef8c87f8abfb884dac04e97824b61299880c43f4ce675dd2cbeadd3c9b466d2"}, - {file = "scipy-1.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b05d43735bb2f07d689f56f7b474788a13ed8adc484a85aa65c0fd931cf9ccd2"}, - {file = "scipy-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:716e389b694c4bb564b4fc0c51bc84d381735e0d39d3f26ec1af2556ec6aad94"}, - {file = "scipy-1.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:631f07b3734d34aced009aaf6fedfd0eb3498a97e581c3b1e5f14a04164a456d"}, - {file = "scipy-1.14.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:af29a935803cc707ab2ed7791c44288a682f9c8107bc00f0eccc4f92c08d6e07"}, - {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:2843f2d527d9eebec9a43e6b406fb7266f3af25a751aa91d62ff416f54170bc5"}, - {file = "scipy-1.14.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:eb58ca0abd96911932f688528977858681a59d61a7ce908ffd355957f7025cfc"}, - {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30ac8812c1d2aab7131a79ba62933a2a76f582d5dbbc695192453dae67ad6310"}, - {file = "scipy-1.14.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f9ea80f2e65bdaa0b7627fb00cbeb2daf163caa015e59b7516395fe3bd1e066"}, - {file = "scipy-1.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:edaf02b82cd7639db00dbff629995ef185c8df4c3ffa71a5562a595765a06ce1"}, - {file = "scipy-1.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:2ff38e22128e6c03ff73b6bb0f85f897d2362f8c052e3b8ad00532198fbdae3f"}, - {file = "scipy-1.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1729560c906963fc8389f6aac023739ff3983e727b1a4d87696b7bf108316a79"}, - {file = "scipy-1.14.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:4079b90df244709e675cdc8b93bfd8a395d59af40b72e339c2287c91860deb8e"}, - {file = "scipy-1.14.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e0cf28db0f24a38b2a0ca33a85a54852586e43cf6fd876365c86e0657cfe7d73"}, - {file = "scipy-1.14.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:0c2f95de3b04e26f5f3ad5bb05e74ba7f68b837133a4492414b3afd79dfe540e"}, - {file = "scipy-1.14.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b99722ea48b7ea25e8e015e8341ae74624f72e5f21fc2abd45f3a93266de4c5d"}, - {file = "scipy-1.14.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5149e3fd2d686e42144a093b206aef01932a0059c2a33ddfa67f5f035bdfe13e"}, - {file = "scipy-1.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e4f5a7c49323533f9103d4dacf4e4f07078f360743dec7f7596949149efeec06"}, - {file = "scipy-1.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:baff393942b550823bfce952bb62270ee17504d02a1801d7fd0719534dfb9c84"}, - {file = "scipy-1.14.1.tar.gz", hash = "sha256:5a275584e726026a5699459aa72f828a610821006228e841b94275c4a7c08417"}, -] - -[package.dependencies] -numpy = ">=1.23.5,<2.3" + {file = "scipy-1.15.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:aeac60d3562a7bf2f35549bdfdb6b1751c50590f55ce7322b4b2fc821dc27fca"}, + {file = "scipy-1.15.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:5abbdc6ede5c5fed7910cf406a948e2c0869231c0db091593a6b2fa78be77e5d"}, + {file = "scipy-1.15.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:eb1533c59f0ec6c55871206f15a5c72d1fae7ad3c0a8ca33ca88f7c309bbbf8c"}, + {file = "scipy-1.15.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:de112c2dae53107cfeaf65101419662ac0a54e9a088c17958b51c95dac5de56d"}, + {file = "scipy-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2240e1fd0782e62e1aacdc7234212ee271d810f67e9cd3b8d521003a82603ef8"}, + {file = "scipy-1.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d35aef233b098e4de88b1eac29f0df378278e7e250a915766786b773309137c4"}, + {file = "scipy-1.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1b29e4fc02e155a5fd1165f1e6a73edfdd110470736b0f48bcbe48083f0eee37"}, + {file = "scipy-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:0e5b34f8894f9904cc578008d1a9467829c1817e9f9cb45e6d6eeb61d2ab7731"}, + {file = "scipy-1.15.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:46e91b5b16909ff79224b56e19cbad65ca500b3afda69225820aa3afbf9ec020"}, + {file = "scipy-1.15.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:82bff2eb01ccf7cea8b6ee5274c2dbeadfdac97919da308ee6d8e5bcbe846443"}, + {file = "scipy-1.15.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:9c8254fe21dd2c6c8f7757035ec0c31daecf3bb3cffd93bc1ca661b731d28136"}, + {file = "scipy-1.15.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:c9624eeae79b18cab1a31944b5ef87aa14b125d6ab69b71db22f0dbd962caf1e"}, + {file = "scipy-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d13bbc0658c11f3d19df4138336e4bce2c4fbd78c2755be4bf7b8e235481557f"}, + {file = "scipy-1.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdca4c7bb8dc41307e5f39e9e5d19c707d8e20a29845e7533b3bb20a9d4ccba0"}, + {file = "scipy-1.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6f376d7c767731477bac25a85d0118efdc94a572c6b60decb1ee48bf2391a73b"}, + {file = "scipy-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:61513b989ee8d5218fbeb178b2d51534ecaddba050db949ae99eeb3d12f6825d"}, + {file = "scipy-1.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5beb0a2200372b7416ec73fdae94fe81a6e85e44eb49c35a11ac356d2b8eccc6"}, + {file = "scipy-1.15.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fde0f3104dfa1dfbc1f230f65506532d0558d43188789eaf68f97e106249a913"}, + {file = "scipy-1.15.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:35c68f7044b4e7ad73a3e68e513dda946989e523df9b062bd3cf401a1a882192"}, + {file = "scipy-1.15.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:52475011be29dfcbecc3dfe3060e471ac5155d72e9233e8d5616b84e2b542054"}, + {file = "scipy-1.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5972e3f96f7dda4fd3bb85906a17338e65eaddfe47f750e240f22b331c08858e"}, + {file = "scipy-1.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe00169cf875bed0b3c40e4da45b57037dc21d7c7bf0c85ed75f210c281488f1"}, + {file = "scipy-1.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:161f80a98047c219c257bf5ce1777c574bde36b9d962a46b20d0d7e531f86863"}, + {file = "scipy-1.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:327163ad73e54541a675240708244644294cb0a65cca420c9c79baeb9648e479"}, + {file = "scipy-1.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0fcb16eb04d84670722ce8d93b05257df471704c913cb0ff9dc5a1c31d1e9422"}, + {file = "scipy-1.15.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:767e8cf6562931f8312f4faa7ddea412cb783d8df49e62c44d00d89f41f9bbe8"}, + {file = "scipy-1.15.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:37ce9394cdcd7c5f437583fc6ef91bd290014993900643fdfc7af9b052d1613b"}, + {file = "scipy-1.15.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6d26f17c64abd6c6c2dfb39920f61518cc9e213d034b45b2380e32ba78fde4c0"}, + {file = "scipy-1.15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e2448acd79c6374583581a1ded32ac71a00c2b9c62dfa87a40e1dd2520be111"}, + {file = "scipy-1.15.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36be480e512d38db67f377add5b759fb117edd987f4791cdf58e59b26962bee4"}, + {file = "scipy-1.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ccb6248a9987193fe74363a2d73b93bc2c546e0728bd786050b7aef6e17db03c"}, + {file = "scipy-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:952d2e9eaa787f0a9e95b6e85da3654791b57a156c3e6609e65cc5176ccfe6f2"}, + {file = "scipy-1.15.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:b1432102254b6dc7766d081fa92df87832ac25ff0b3d3a940f37276e63eb74ff"}, + {file = "scipy-1.15.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:4e08c6a36f46abaedf765dd2dfcd3698fa4bd7e311a9abb2d80e33d9b2d72c34"}, + {file = "scipy-1.15.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:ec915cd26d76f6fc7ae8522f74f5b2accf39546f341c771bb2297f3871934a52"}, + {file = "scipy-1.15.0-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:351899dd2a801edd3691622172bc8ea01064b1cada794f8641b89a7dc5418db6"}, + {file = "scipy-1.15.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9baff912ea4f78a543d183ed6f5b3bea9784509b948227daaf6f10727a0e2e5"}, + {file = "scipy-1.15.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cd9d9198a7fd9a77f0eb5105ea9734df26f41faeb2a88a0e62e5245506f7b6df"}, + {file = "scipy-1.15.0-cp313-cp313t-win_amd64.whl", hash = "sha256:129f899ed275c0515d553b8d31696924e2ca87d1972421e46c376b9eb87de3d2"}, + {file = "scipy-1.15.0.tar.gz", hash = "sha256:300742e2cc94e36a2880ebe464a1c8b4352a7b0f3e36ec3d2ac006cdbe0219ac"}, +] + +[package.dependencies] +numpy = ">=1.23.5,<2.5" [package.extras] dev = ["cython-lint (>=0.12.2)", "doit (>=0.36.0)", "mypy (==1.10.0)", "pycodestyle", "pydevtool", "rich-click", "ruff (>=0.0.292)", "types-psutil", "typing_extensions"] -doc = ["jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.13.1)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<=7.3.7)", "sphinx-design (>=0.4.0)"] -test = ["Cython", "array-api-strict (>=2.0)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] +doc = ["intersphinx_registry", "jupyterlite-pyodide-kernel", "jupyterlite-sphinx (>=0.16.5)", "jupytext", "matplotlib (>=3.5)", "myst-nb", "numpydoc", "pooch", "pydata-sphinx-theme (>=0.15.2)", "sphinx (>=5.0.0,<8.0.0)", "sphinx-copybutton", "sphinx-design (>=0.4.0)"] +test = ["Cython", "array-api-strict (>=2.0,<2.1.1)", "asv", "gmpy2", "hypothesis (>=6.30)", "meson", "mpmath", "ninja", "pooch", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "scikit-umfpack", "threadpoolctl"] [[package]] name = "sentry-sdk" @@ -9105,23 +9104,23 @@ tornado = ["tornado (>=5)"] [[package]] name = "setuptools" -version = "75.6.0" +version = "75.7.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" files = [ - {file = "setuptools-75.6.0-py3-none-any.whl", hash = "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d"}, - {file = "setuptools-75.6.0.tar.gz", hash = "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6"}, + {file = "setuptools-75.7.0-py3-none-any.whl", hash = "sha256:84fb203f278ebcf5cd08f97d3fb96d3fbed4b629d500b29ad60d11e00769b183"}, + {file = "setuptools-75.7.0.tar.gz", hash = "sha256:886ff7b16cd342f1d1defc16fc98c9ce3fde69e087a4e1983d7ab634e5f41f4f"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.7.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12,<1.14)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "sgmllib3k" @@ -9532,13 +9531,13 @@ test = ["pytest", "tornado (>=4.5)", "typeguard"] [[package]] name = "tencentcloud-sdk-python-common" -version = "3.0.1294" +version = "3.0.1298" description = "Tencent Cloud Common SDK for Python" optional = false python-versions = "*" files = [ - {file = "tencentcloud-sdk-python-common-3.0.1294.tar.gz", hash = "sha256:a6d079690b69d60e8bfd1e27a65138e36d7f6cc57de7e7549c45a6084bc4743c"}, - {file = "tencentcloud_sdk_python_common-3.0.1294-py2.py3-none-any.whl", hash = "sha256:bc43fb56e6a9d0f825d74f1cbdf159e0417ff3f4b59b9c75a73eeb6526259329"}, + {file = "tencentcloud-sdk-python-common-3.0.1298.tar.gz", hash = "sha256:0f0f182410c1ceda5764ff8bcbef27aa6139caf1c5f5985d94ec731a41c8a59f"}, + {file = "tencentcloud_sdk_python_common-3.0.1298-py2.py3-none-any.whl", hash = "sha256:c80929a0ff57ebee4ceec749dc82d5f2d1105b888e55175a7e9c722afc3a5d7a"}, ] [package.dependencies] @@ -9546,17 +9545,17 @@ requests = ">=2.16.0" [[package]] name = "tencentcloud-sdk-python-hunyuan" -version = "3.0.1294" +version = "3.0.1298" description = "Tencent Cloud Hunyuan SDK for Python" optional = false python-versions = "*" files = [ - {file = "tencentcloud-sdk-python-hunyuan-3.0.1294.tar.gz", hash = "sha256:ca7463b26e54bd4dc922c5bce24f728b9fed1494d55a3a0a76594db74f347657"}, - {file = "tencentcloud_sdk_python_hunyuan-3.0.1294-py2.py3-none-any.whl", hash = "sha256:b53ea5c7880623d649eb235a2a6865312db1276b03bf21d9520d2136d14dadf4"}, + {file = "tencentcloud-sdk-python-hunyuan-3.0.1298.tar.gz", hash = "sha256:c3d86a577de02046d25682a3804955453555fa641082bb8765238460bded3f03"}, + {file = "tencentcloud_sdk_python_hunyuan-3.0.1298-py2.py3-none-any.whl", hash = "sha256:f01e33318b6a4152ac88c500fda77f2cda1864eeca000cdd29c41e4f92f8de65"}, ] [package.dependencies] -tencentcloud-sdk-python-common = "3.0.1294" +tencentcloud-sdk-python-common = "3.0.1298" [[package]] name = "termcolor" @@ -10107,13 +10106,13 @@ files = [ [[package]] name = "unstructured" -version = "0.16.11" +version = "0.16.12" description = "A library that prepares raw documents for downstream ML tasks." optional = false python-versions = "<3.13,>=3.9.0" files = [ - {file = "unstructured-0.16.11-py3-none-any.whl", hash = "sha256:a92d5bc2c2b7bb23369641fb7a7f0daba1775639199306ce4cd83ca564a03763"}, - {file = "unstructured-0.16.11.tar.gz", hash = "sha256:33ebf68aae11ce33c8a96335296557b5abd8ba96eaba3e5a1554c0b9eee40bb5"}, + {file = "unstructured-0.16.12-py3-none-any.whl", hash = "sha256:bcac29ac1b38fba4228c5a1a7721d1aa7c48220f7c1dd43b563645c56e978c49"}, + {file = "unstructured-0.16.12.tar.gz", hash = "sha256:c3133731c6edb9c2f474e62cb2b560cd0a8d578c4532ec14d8c0941e401770b0"}, ] [package.dependencies] @@ -10127,6 +10126,7 @@ html5lib = "*" langdetect = "*" lxml = "*" markdown = {version = "*", optional = true, markers = "extra == \"md\""} +ndjson = "*" nltk = "*" numpy = "<2" psutil = "*" @@ -10998,13 +10998,13 @@ requests = "*" [[package]] name = "zhipuai" -version = "2.1.5.20241204" +version = "2.1.5.20250106" description = "A SDK library for accessing big model apis from ZhipuAI" optional = false python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" files = [ - {file = "zhipuai-2.1.5.20241204-py3-none-any.whl", hash = "sha256:063c7527d6741ced82eedb19d53fd24ce61cf43ab835ee3c0262843f59503a7c"}, - {file = "zhipuai-2.1.5.20241204.tar.gz", hash = "sha256:888b42a83c8f1daf07375b84e560219eedab96b9f9e59542f0329928291db635"}, + {file = "zhipuai-2.1.5.20250106-py3-none-any.whl", hash = "sha256:ca76095f32db501e36038fc1ac4b287b88ed90c4cdd28902d3b1a9365fff879b"}, + {file = "zhipuai-2.1.5.20250106.tar.gz", hash = "sha256:45d391be336a210b360f126443f07882fa6d8184a148c46a8c7d0b7607d6d1f8"}, ] [package.dependencies] @@ -11220,4 +11220,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.13" -content-hash = "8c74132f2fe0b8dc7318bfbfb1bd3dbf7cd2ecfd4fc430c8924b46edacc8d33e" +content-hash = "907718f7ca775ad226c1f668f4bb6c6dbfa6cacc556fce43a8ad0b6f3c35095a" diff --git a/api/pyproject.toml b/api/pyproject.toml index 1738899e81627a..f8c6f599d182b0 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -71,7 +71,7 @@ pyjwt = "~2.8.0" pypdfium2 = "~4.30.0" python = ">=3.11,<3.13" python-docx = "~1.1.0" -python-dotenv = "1.0.0" +python-dotenv = "1.0.1" pyyaml = "~6.0.1" readabilipy = "0.2.0" redis = { version = "~5.0.3", extras = ["hiredis"] } @@ -157,7 +157,7 @@ opensearch-py = "2.4.0" oracledb = "~2.2.1" pgvecto-rs = { version = "~0.2.1", extras = ['sqlalchemy'] } pgvector = "0.2.5" -pymilvus = "~2.4.4" +pymilvus = "~2.5.0" pymochow = "1.3.1" pyobvector = "~0.1.6" qdrant-client = "1.7.3" diff --git a/api/tests/integration_tests/vdb/milvus/test_milvus.py b/api/tests/integration_tests/vdb/milvus/test_milvus.py index c99739a86360e2..0e13f9369e8c94 100644 --- a/api/tests/integration_tests/vdb/milvus/test_milvus.py +++ b/api/tests/integration_tests/vdb/milvus/test_milvus.py @@ -19,9 +19,9 @@ def __init__(self): ) def search_by_full_text(self): - # milvus dos not support full text searching yet in < 2.3.x + # milvus support BM25 full text search after version 2.5.0-beta hits_by_full_text = self.vector.search_by_full_text(query=get_example_text()) - assert len(hits_by_full_text) == 0 + assert len(hits_by_full_text) >= 0 def get_ids_by_metadata_field(self): ids = self.vector.get_ids_by_metadata_field(key="document_id", value=self.example_doc_id) diff --git a/docker/.env.example b/docker/.env.example index 85277027b663c2..333050a8929bf2 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -403,6 +403,7 @@ MILVUS_URI=http://127.0.0.1:19530 MILVUS_TOKEN= MILVUS_USER=root MILVUS_PASSWORD=Milvus +MILVUS_ENABLE_HYBRID_SEARCH=False # MyScale configuration, only available when VECTOR_STORE is `myscale` # For multi-language support, please set MYSCALE_FTS_PARAMS with referring to: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 29e312de7f9c8b..173a88bc4cd933 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -138,6 +138,7 @@ x-shared-env: &shared-api-worker-env MILVUS_TOKEN: ${MILVUS_TOKEN:-} MILVUS_USER: ${MILVUS_USER:-root} MILVUS_PASSWORD: ${MILVUS_PASSWORD:-Milvus} + MILVUS_ENABLE_HYBRID_SEARCH: ${MILVUS_ENABLE_HYBRID_SEARCH:-False} MYSCALE_HOST: ${MYSCALE_HOST:-myscale} MYSCALE_PORT: ${MYSCALE_PORT:-8123} MYSCALE_USER: ${MYSCALE_USER:-default} @@ -799,7 +800,7 @@ services: milvus-standalone: container_name: milvus-standalone - image: milvusdb/milvus:v2.3.1 + image: milvusdb/milvus:v2.5.0-beta profiles: - milvus command: [ 'milvus', 'run', 'standalone' ] From bee32d960a1fd24a6f267b9463144ad3b4f885f8 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:26:05 +0800 Subject: [PATCH 14/42] fix #12453 #12482 (#12495) --- .../console/datasets/datasets_document.py | 3 ++- api/services/dataset_service.py | 24 +++++++++++++------ api/tasks/deal_dataset_vector_index_task.py | 5 +++- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index 5a3c6f843290b8..c11beaeee1035c 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -257,7 +257,8 @@ def post(self, dataset_id): parser.add_argument("original_document_id", type=str, required=False, location="json") parser.add_argument("doc_form", type=str, default="text_model", required=False, nullable=False, location="json") parser.add_argument("retrieval_model", type=dict, required=False, nullable=False, location="json") - + parser.add_argument("embedding_model", type=str, required=False, nullable=True, location="json") + parser.add_argument("embedding_model_provider", type=str, required=False, nullable=True, location="json") parser.add_argument( "doc_language", type=str, default="English", required=False, nullable=False, location="json" ) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 3be8a38b031d9c..82433b64ffcc3a 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -792,13 +792,19 @@ def save_document_with_dataset_id( dataset.indexing_technique = knowledge_config.indexing_technique if knowledge_config.indexing_technique == "high_quality": model_manager = ModelManager() - embedding_model = model_manager.get_default_model_instance( - tenant_id=current_user.current_tenant_id, model_type=ModelType.TEXT_EMBEDDING - ) - dataset.embedding_model = embedding_model.model - dataset.embedding_model_provider = embedding_model.provider + if knowledge_config.embedding_model and knowledge_config.embedding_model_provider: + dataset_embedding_model = knowledge_config.embedding_model + dataset_embedding_model_provider = knowledge_config.embedding_model_provider + else: + embedding_model = model_manager.get_default_model_instance( + tenant_id=current_user.current_tenant_id, model_type=ModelType.TEXT_EMBEDDING + ) + dataset_embedding_model = embedding_model.model + dataset_embedding_model_provider = embedding_model.provider + dataset.embedding_model = dataset_embedding_model + dataset.embedding_model_provider = dataset_embedding_model_provider dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding( - embedding_model.provider, embedding_model.model + dataset_embedding_model_provider, dataset_embedding_model ) dataset.collection_binding_id = dataset_collection_binding.id if not dataset.retrieval_model: @@ -810,7 +816,11 @@ def save_document_with_dataset_id( "score_threshold_enabled": False, } - dataset.retrieval_model = knowledge_config.retrieval_model.model_dump() or default_retrieval_model # type: ignore + dataset.retrieval_model = ( + knowledge_config.retrieval_model.model_dump() + if knowledge_config.retrieval_model + else default_retrieval_model + ) # type: ignore documents = [] if knowledge_config.original_document_id: diff --git a/api/tasks/deal_dataset_vector_index_task.py b/api/tasks/deal_dataset_vector_index_task.py index 0efc924a77aae4..a9b5ab91a8934c 100644 --- a/api/tasks/deal_dataset_vector_index_task.py +++ b/api/tasks/deal_dataset_vector_index_task.py @@ -28,7 +28,7 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str): if not dataset: raise Exception("Dataset not found") - index_type = dataset.doc_form + index_type = dataset.doc_form or IndexType.PARAGRAPH_INDEX index_processor = IndexProcessorFactory(index_type).init_index_processor() if action == "remove": index_processor.clean(dataset, None, with_keywords=False) @@ -157,6 +157,9 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str): {"indexing_status": "error", "error": str(e)}, synchronize_session=False ) db.session.commit() + else: + # clean collection + index_processor.clean(dataset, None, with_keywords=False, delete_child_chunks=False) end_at = time.perf_counter() logging.info( From f4ee50a7ad72d8838417169b1dd9621b9fd9833b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9D=9E=E6=B3=95=E6=93=8D=E4=BD=9C?= Date: Wed, 8 Jan 2025 18:37:12 +0800 Subject: [PATCH 15/42] chore: improve app doc (#12490) --- web/app/components/develop/doc.tsx | 18 +++++++++++ .../develop/template/template.en.mdx | 22 +++---------- .../develop/template/template.ja.mdx | 22 +++---------- .../develop/template/template.zh.mdx | 21 ++---------- .../template/template_advanced_chat.en.mdx | 32 ++++--------------- .../template/template_advanced_chat.ja.mdx | 32 ++++--------------- .../template/template_advanced_chat.zh.mdx | 32 +++---------------- .../develop/template/template_chat.en.mdx | 24 ++++---------- .../develop/template/template_chat.ja.mdx | 32 ++++--------------- .../develop/template/template_chat.zh.mdx | 32 +++---------------- .../develop/template/template_workflow.en.mdx | 22 +++---------- .../develop/template/template_workflow.ja.mdx | 22 +++---------- .../develop/template/template_workflow.zh.mdx | 23 +++---------- 13 files changed, 75 insertions(+), 259 deletions(-) diff --git a/web/app/components/develop/doc.tsx b/web/app/components/develop/doc.tsx index d3076c4e746b4f..fd524ef0c16f6f 100644 --- a/web/app/components/develop/doc.tsx +++ b/web/app/components/develop/doc.tsx @@ -61,6 +61,23 @@ const Doc = ({ appDetail }: IDocProps) => { // Run after component has rendered setTimeout(extractTOC, 0) }, [appDetail, locale]) + + const handleTocClick = (e: React.MouseEvent, item: { href: string; text: string }) => { + e.preventDefault() + const targetId = item.href.replace('#', '') + const element = document.getElementById(targetId) + if (element) { + const scrollContainer = document.querySelector('.overflow-auto') + if (scrollContainer) { + const headerOffset = 80 + const elementTop = element.offsetTop - headerOffset + scrollContainer.scrollTo({ + top: elementTop, + behavior: 'smooth', + }) + } + } + } return (
@@ -82,6 +99,7 @@ const Doc = ({ appDetail }: IDocProps) => { handleTocClick(e, item)} > {item.text} diff --git a/web/app/components/develop/template/template.en.mdx b/web/app/components/develop/template/template.en.mdx index 877955039c82ca..ea3b7f3d235051 100755 --- a/web/app/components/develop/template/template.en.mdx +++ b/web/app/components/develop/template/template.en.mdx @@ -444,22 +444,16 @@ The text generation application offers non-session support and is ideal for tran Used to get basic information about this application - ### Query - - - User identifier, defined by the developer's rules, must be unique within the application. - - ### Response - `name` (string) application name - `description` (string) application description - `tags` (array[string]) application tags - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -490,14 +484,6 @@ The text generation application offers non-session support and is ideal for tran Used at the start of entering the page to obtain information such as features, input parameter names, types, and default values. - ### Query - - - - User identifier, defined by the developer's rules, must be unique within the application. - - - ### Response - `opening_statement` (string) Opening statement - `suggested_questions` (array[string]) List of suggested questions for the opening @@ -541,10 +527,10 @@ The text generation application offers non-session support and is ideal for tran - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template.ja.mdx b/web/app/components/develop/template/template.ja.mdx index c3b376f2e743c2..4670404277584d 100755 --- a/web/app/components/develop/template/template.ja.mdx +++ b/web/app/components/develop/template/template.ja.mdx @@ -442,22 +442,16 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from このアプリケーションの基本情報を取得するために使用されます - ### Query - - - ユーザー識別子、開発者のルールによって定義され、アプリケーション内で一意でなければなりません。 - - ### Response - `name` (string) アプリケーションの名前 - `description` (string) アプリケーションの説明 - `tags` (array[string]) アプリケーションのタグ - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -488,14 +482,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from ページ開始時に、機能、入力パラメータ名、タイプ、デフォルト値などの情報を取得するために使用されます。 - ### クエリ - - - - 開発者のルールで定義されたユーザー識別子。アプリケーション内で一意である必要があります。 - - - ### レスポンス - `opening_statement` (string) 開始文 - `suggested_questions` (array[string]) 開始時の提案質問リスト @@ -539,10 +525,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template.zh.mdx b/web/app/components/develop/template/template.zh.mdx index be7470480f36df..d8cabb536d4dc0 100755 --- a/web/app/components/develop/template/template.zh.mdx +++ b/web/app/components/develop/template/template.zh.mdx @@ -419,22 +419,15 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 用于获取应用的基本信息 - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - ### Response - `name` (string) 应用名称 - `description` (string) 应用描述 - `tags` (array[string]) 应用标签 - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -465,14 +458,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 用于进入页面一开始，获取功能开关、输入参数名称、类型及默认值等使用。 - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - - ### Response - `opening_statement` (string) 开场白 - `suggested_questions` (array[string]) 开场推荐问题列表 @@ -518,7 +503,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template_advanced_chat.en.mdx b/web/app/components/develop/template/template_advanced_chat.en.mdx index c7b92acc52a3d9..e905e9e0c62aa7 100644 --- a/web/app/components/develop/template/template_advanced_chat.en.mdx +++ b/web/app/components/develop/template/template_advanced_chat.en.mdx @@ -952,22 +952,16 @@ Chat applications support session persistence, allowing previous chat history to Used to get basic information about this application - ### Query - - - User identifier, defined by the developer's rules, must be unique within the application. - - ### Response - `name` (string) application name - `description` (string) application description - `tags` (array[string]) application tags - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -998,14 +992,6 @@ Chat applications support session persistence, allowing previous chat history to Used at the start of entering the page to obtain information such as features, input parameter names, types, and default values. - ### Query - - - - User identifier, defined by the developer's rules, must be unique within the application. - - - ### Response - `opening_statement` (string) Opening statement - `suggested_questions` (array[string]) List of suggested questions for the opening @@ -1049,10 +1035,10 @@ Chat applications support session persistence, allowing previous chat history to - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` @@ -1117,13 +1103,7 @@ Chat applications support session persistence, allowing previous chat history to Used to get icons of tools in this application - ### Query - - - User identifier, defined by the developer's rules, must be unique within the application. - - ### Response - `tool_icons`(object[string]) tool icons - `tool_name` (string) @@ -1134,9 +1114,9 @@ Chat applications support session persistence, allowing previous chat history to - (string) url of icon - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/meta?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/meta' \ -H 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template_advanced_chat.ja.mdx b/web/app/components/develop/template/template_advanced_chat.ja.mdx index 67e8d8f7fec888..08255b88823831 100644 --- a/web/app/components/develop/template/template_advanced_chat.ja.mdx +++ b/web/app/components/develop/template/template_advanced_chat.ja.mdx @@ -951,22 +951,16 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from このアプリケーションの基本情報を取得するために使用されます - ### Query - - - ユーザー識別子、開発者のルールによって定義され、アプリケーション内で一意でなければなりません。 - - ### Response - `name` (string) アプリケーションの名前 - `description` (string) アプリケーションの説明 - `tags` (array[string]) アプリケーションのタグ - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -997,14 +991,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from ページに入る際に、機能、入力パラメータ名、タイプ、デフォルト値などの情報を取得するために使用されます。 - ### クエリ - - - - ユーザー識別子、開発者のルールによって定義され、アプリケーション内で一意でなければなりません。 - - - ### 応答 - `opening_statement` (string) 開始の挨拶 - `suggested_questions` (array[string]) 開始時の推奨質問のリスト @@ -1048,10 +1034,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` @@ -1116,13 +1102,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from このアプリケーションのツールのアイコンを取得するために使用されます - ### クエリ - - - ユーザー識別子、開発者のルールによって定義され、アプリケーション内で一意でなければなりません。 - - ### 応答 - `tool_icons`(object[string]) ツールアイコン - `tool_name` (string) @@ -1133,9 +1113,9 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - (string) アイコンのURL - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/meta?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/meta' \ -H 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template_advanced_chat.zh.mdx b/web/app/components/develop/template/template_advanced_chat.zh.mdx index 3bb17d850299a0..8d181d7e0b7581 100755 --- a/web/app/components/develop/template/template_advanced_chat.zh.mdx +++ b/web/app/components/develop/template/template_advanced_chat.zh.mdx @@ -985,22 +985,15 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 用于获取应用的基本信息 - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - ### Response - `name` (string) 应用名称 - `description` (string) 应用描述 - `tags` (array[string]) 应用标签 - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -1031,14 +1024,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 用于进入页面一开始，获取功能开关、输入参数名称、类型及默认值等使用。 - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - - ### Response - `opening_statement` (string) 开场白 - `suggested_questions` (array[string]) 开场推荐问题列表 @@ -1084,7 +1069,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` @@ -1141,13 +1126,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 用于获取工具icon - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - ### Response - `tool_icons`(object[string]) 工具图标 - `工具名称` (string) @@ -1158,9 +1136,9 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' - (string) 图标URL - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/meta?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/meta' \ -H 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template_chat.en.mdx b/web/app/components/develop/template/template_chat.en.mdx index d38e80407add80..7de329a7a0ed0e 100644 --- a/web/app/components/develop/template/template_chat.en.mdx +++ b/web/app/components/develop/template/template_chat.en.mdx @@ -980,22 +980,16 @@ Chat applications support session persistence, allowing previous chat history to Used to get basic information about this application - ### Query - - - User identifier, defined by the developer's rules, must be unique within the application. - - ### Response - `name` (string) application name - `description` (string) application description - `tags` (array[string]) application tags - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -1077,10 +1071,10 @@ Chat applications support session persistence, allowing previous chat history to - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` @@ -1145,13 +1139,7 @@ Chat applications support session persistence, allowing previous chat history to Used to get icons of tools in this application - ### Query - - - User identifier, defined by the developer's rules, must be unique within the application. - - ### Response - `tool_icons`(object[string]) tool icons - `tool_name` (string) @@ -1162,9 +1150,9 @@ Chat applications support session persistence, allowing previous chat history to - (string) url of icon - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/meta?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/meta' \ -H 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template_chat.ja.mdx b/web/app/components/develop/template/template_chat.ja.mdx index 96db9912d5da71..8107f6260ada30 100644 --- a/web/app/components/develop/template/template_chat.ja.mdx +++ b/web/app/components/develop/template/template_chat.ja.mdx @@ -978,22 +978,16 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from このアプリケーションの基本情報を取得するために使用されます - ### Query - - - ユーザー識別子、開発者のルールによって定義され、アプリケーション内で一意でなければなりません。 - - ### Response - `name` (string) アプリケーションの名前 - `description` (string) アプリケーションの説明 - `tags` (array[string]) アプリケーションのタグ - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -1024,14 +1018,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from ページに入る際に、機能、入力パラメータ名、タイプ、デフォルト値などの情報を取得するために使用されます。 - ### クエリ - - - - ユーザー識別子、開発者のルールで定義され、アプリケーション内で一意でなければなりません。 - - - ### 応答 - `opening_statement` (string) 開始文 - `suggested_questions` (array[string]) 開始時の推奨質問のリスト @@ -1075,10 +1061,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` @@ -1143,13 +1129,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from このアプリケーションのツールのアイコンを取得するために使用されます - ### クエリ - - - ユーザー識別子、開発者のルールで定義され、アプリケーション内で一意でなければなりません。 - - ### 応答 - `tool_icons`(object[string]) ツールアイコン - `tool_name` (string) @@ -1160,9 +1140,9 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - (string) アイコンのURL - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/meta?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/meta' \ -H 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template_chat.zh.mdx b/web/app/components/develop/template/template_chat.zh.mdx index 3d6e3630be5129..ec845970b4e958 100644 --- a/web/app/components/develop/template/template_chat.zh.mdx +++ b/web/app/components/develop/template/template_chat.zh.mdx @@ -993,22 +993,15 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 用于获取应用的基本信息 - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - ### Response - `name` (string) 应用名称 - `description` (string) 应用描述 - `tags` (array[string]) 应用标签 - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -1039,14 +1032,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 用于进入页面一开始，获取功能开关、输入参数名称、类型及默认值等使用。 - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - - ### Response - `opening_statement` (string) 开场白 - `suggested_questions` (array[string]) 开场推荐问题列表 @@ -1092,7 +1077,7 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` @@ -1149,13 +1134,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' 用于获取工具icon - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - ### Response - `tool_icons`(object[string]) 工具图标 - `工具名称` (string) @@ -1166,9 +1144,9 @@ import { Row, Col, Properties, Property, Heading, SubProperty } from '../md.tsx' - (string) 图标URL - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/meta?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/meta' \ -H 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template_workflow.en.mdx b/web/app/components/develop/template/template_workflow.en.mdx index 58c533c60b1375..27c0d26505156e 100644 --- a/web/app/components/develop/template/template_workflow.en.mdx +++ b/web/app/components/develop/template/template_workflow.en.mdx @@ -610,22 +610,16 @@ Workflow applications offers non-session support and is ideal for translation, a Used to get basic information about this application - ### Query - - - User identifier, defined by the developer's rules, must be unique within the application. - - ### Response - `name` (string) application name - `description` (string) application description - `tags` (array[string]) application tags - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -656,14 +650,6 @@ Workflow applications offers non-session support and is ideal for translation, a Used at the start of entering the page to obtain information such as features, input parameter names, types, and default values. - ### Query - - - - User identifier, defined by the developer's rules, must be unique within the application. - - - ### Response - `user_input_form` (array[object]) User input form configuration - `text-input` (object) Text input control @@ -697,10 +683,10 @@ Workflow applications offers non-session support and is ideal for translation, a - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template_workflow.ja.mdx b/web/app/components/develop/template/template_workflow.ja.mdx index 2653b4913d7d06..9e66973db74ec0 100644 --- a/web/app/components/develop/template/template_workflow.ja.mdx +++ b/web/app/components/develop/template/template_workflow.ja.mdx @@ -610,22 +610,16 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from このアプリケーションの基本情報を取得するために使用されます - ### Query - - - ユーザー識別子、開発者のルールによって定義され、アプリケーション内で一意でなければなりません。 - - ### Response - `name` (string) アプリケーションの名前 - `description` (string) アプリケーションの説明 - `tags` (array[string]) アプリケーションのタグ - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -656,14 +650,6 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from ページに入る際に、機能、入力パラメータ名、タイプ、デフォルト値などの情報を取得するために使用されます。 - ### クエリ - - - - ユーザー識別子、開発者のルールで定義され、アプリケーション内で一意でなければなりません。 - - - ### 応答 - `user_input_form` (array[object]) ユーザー入力フォームの設定 - `text-input` (object) テキスト入力コントロール @@ -697,10 +683,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` diff --git a/web/app/components/develop/template/template_workflow.zh.mdx b/web/app/components/develop/template/template_workflow.zh.mdx index ddffc0f02d5070..40dfb863a0a6b2 100644 --- a/web/app/components/develop/template/template_workflow.zh.mdx +++ b/web/app/components/develop/template/template_workflow.zh.mdx @@ -602,22 +602,15 @@ Workflow 应用无会话支持，适合用于翻译/文章写作/总结 AI 等 用于获取应用的基本信息 - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - ### Response - `name` (string) 应用名称 - `description` (string) 应用描述 - `tags` (array[string]) 应用标签 - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/info?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/info' \ -H 'Authorization: Bearer {api_key}' ``` @@ -648,14 +641,6 @@ Workflow 应用无会话支持，适合用于翻译/文章写作/总结 AI 等 用于进入页面一开始，获取功能开关、输入参数名称、类型及默认值等使用。 - ### Query - - - - 用户标识，由开发者定义规则，需保证用户标识在应用内唯一。 - - - ### Response - `user_input_form` (array[object]) 用户输入表单配置 - `text-input` (object) 文本输入控件 @@ -689,10 +674,10 @@ Workflow 应用无会话支持，适合用于翻译/文章写作/总结 AI 等 - + ```bash {{ title: 'cURL' }} - curl -X GET '${props.appDetail.api_base_url}/parameters?user=abc-123' \ + curl -X GET '${props.appDetail.api_base_url}/parameters' \ --header 'Authorization: Bearer {api_key}' ``` From 1b940e7daa748e549ac2f928623170fa16ef1810 Mon Sep 17 00:00:00 2001 From: kurokobo Date: Thu, 9 Jan 2025 01:04:58 +0900 Subject: [PATCH 16/42] feat: add ci job to test template for docker compose (#12514) --- .github/workflows/style.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index b5e63a8870baa8..12213380bdcf52 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -82,6 +82,33 @@ jobs: if: steps.changed-files.outputs.any_changed == 'true' run: yarn run lint + docker-compose-template: + name: Docker Compose Template + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + files: | + docker/generate_docker_compose + docker/.env.example + docker/docker-compose-template.yaml + docker/docker-compose.yaml + + - name: Generate Docker Compose + if: steps.changed-files.outputs.any_changed == 'true' + run: | + cd docker + ./generate_docker_compose + + - name: Check for changes + if: steps.changed-files.outputs.any_changed == 'true' + run: git diff --exit-code superlinter: name: SuperLinter From b4c1c2f73100c942090bff951d8c5125e55fa4ec Mon Sep 17 00:00:00 2001 From: Hiroshi Fujita Date: Thu, 9 Jan 2025 11:21:22 +0900 Subject: [PATCH 17/42] fix: Reverse sync docker-compose-template.yaml (#12509) --- docker/.env.example | 2 +- docker/docker-compose-template.yaml | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 333050a8929bf2..b21bdc70853483 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -513,7 +513,7 @@ TENCENT_VECTOR_DB_SHARD=1 TENCENT_VECTOR_DB_REPLICAS=2 # ElasticSearch configuration, only available when VECTOR_STORE is `elasticsearch` -ELASTICSEARCH_HOST=elasticsearch +ELASTICSEARCH_HOST=0.0.0.0 ELASTICSEARCH_PORT=9200 ELASTICSEARCH_USERNAME=elastic ELASTICSEARCH_PASSWORD=elastic diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index c96b0538cad61d..6d70f14424ff1a 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -409,7 +409,7 @@ services: milvus-standalone: container_name: milvus-standalone - image: milvusdb/milvus:v2.3.1 + image: milvusdb/milvus:v2.5.0-beta profiles: - milvus command: [ 'milvus', 'run', 'standalone' ] @@ -493,20 +493,28 @@ services: container_name: elasticsearch profiles: - elasticsearch + - elasticsearch-ja restart: always volumes: + - ./elasticsearch/docker-entrypoint.sh:/docker-entrypoint-mount.sh - dify_es01_data:/usr/share/elasticsearch/data environment: ELASTIC_PASSWORD: ${ELASTICSEARCH_PASSWORD:-elastic} + VECTOR_STORE: ${VECTOR_STORE:-} cluster.name: dify-es-cluster node.name: dify-es0 discovery.type: single-node - xpack.license.self_generated.type: trial + xpack.license.self_generated.type: basic xpack.security.enabled: 'true' xpack.security.enrollment.enabled: 'false' xpack.security.http.ssl.enabled: 'false' ports: - ${ELASTICSEARCH_PORT:-9200}:9200 + deploy: + resources: + limits: + memory: 2g + entrypoint: [ 'sh', '-c', "sh /docker-entrypoint-mount.sh" ] healthcheck: test: [ 'CMD', 'curl', '-s', 'http://localhost:9200/_cluster/health?pretty' ] interval: 30s From b7a4e3903e4e3ab2e2c1d4d8ca8dd6879ef0712e Mon Sep 17 00:00:00 2001 From: NFish Date: Thu, 9 Jan 2025 10:40:45 +0800 Subject: [PATCH 18/42] fix: add last_refresh_time to track the validity of is_other_tab_refreshing (#12517) --- web/service/refresh-token.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/web/service/refresh-token.ts b/web/service/refresh-token.ts index b193779629e26e..f42b10cc8eb0ff 100644 --- a/web/service/refresh-token.ts +++ b/web/service/refresh-token.ts @@ -21,16 +21,23 @@ function waitUntilTokenRefreshed() { }) } +const isRefreshingSignAvailable = function (delta: number) { + const nowTime = new Date().getTime() + const lastTime = globalThis.localStorage.getItem('last_refresh_time') || '0' + return nowTime - parseInt(lastTime) <= delta +} + // only one request can send -async function getNewAccessToken(): Promise { +async function getNewAccessToken(timeout: number): Promise { try { const isRefreshingSign = globalThis.localStorage.getItem(LOCAL_STORAGE_KEY) - if ((isRefreshingSign && isRefreshingSign === '1') || isRefreshing) { + if ((isRefreshingSign && isRefreshingSign === '1' && isRefreshingSignAvailable(timeout)) || isRefreshing) { await waitUntilTokenRefreshed() } else { isRefreshing = true globalThis.localStorage.setItem(LOCAL_STORAGE_KEY, '1') + globalThis.localStorage.setItem('last_refresh_time', new Date().getTime().toString()) globalThis.addEventListener('beforeunload', releaseRefreshLock) const refresh_token = globalThis.localStorage.getItem('refresh_token') @@ -72,6 +79,7 @@ function releaseRefreshLock() { if (isRefreshing) { isRefreshing = false globalThis.localStorage.removeItem(LOCAL_STORAGE_KEY) + globalThis.localStorage.removeItem('last_refresh_time') globalThis.removeEventListener('beforeunload', releaseRefreshLock) } } @@ -80,5 +88,5 @@ export async function refreshAccessTokenOrRelogin(timeout: number) { return Promise.race([new Promise((resolve, reject) => setTimeout(() => { releaseRefreshLock() reject(new Error('request timeout')) - }, timeout)), getNewAccessToken()]) + }, timeout)), getNewAccessToken(timeout)]) } From dbe7a7c4fd96330005fc81864a50fe1f84ac8bda Mon Sep 17 00:00:00 2001 From: Gen Sato <52241300+halogen22@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:37:46 +0900 Subject: [PATCH 19/42] Fix: Add a INFO-level log when fallback to gpt2tokenizer (#12508) --- .../model_providers/__base/tokenizers/gpt2_tokenzier.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py index 7f3c4a61e45b06..2f6f4fbbeff722 100644 --- a/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py +++ b/api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py @@ -1,6 +1,9 @@ +import logging from threading import Lock from typing import Any +logger = logging.getLogger(__name__) + _tokenizer: Any = None _lock = Lock() @@ -43,5 +46,6 @@ def get_encoder() -> Any: base_path = abspath(__file__) gpt2_tokenizer_path = join(dirname(base_path), "gpt2") _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path) + logger.info("Fallback to Transformers' GPT-2 tokenizer from tiktoken") return _tokenizer From 20f090537ff0071975dccd158d8b10a86968d6cb Mon Sep 17 00:00:00 2001 From: eux Date: Thu, 9 Jan 2025 14:52:09 +0800 Subject: [PATCH 20/42] feat: add GET upload file API endpoint to dataset service api (#11899) --- api/controllers/service_api/__init__.py | 2 +- .../service_api/dataset/upload_file.py | 54 +++++++++++++++++++ .../datasets/template/template.en.mdx | 51 ++++++++++++++++++ .../datasets/template/template.zh.mdx | 51 ++++++++++++++++++ 4 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 api/controllers/service_api/dataset/upload_file.py diff --git a/api/controllers/service_api/__init__.py b/api/controllers/service_api/__init__.py index d6ab96c329f335..aba9e3ecbb5a47 100644 --- a/api/controllers/service_api/__init__.py +++ b/api/controllers/service_api/__init__.py @@ -7,4 +7,4 @@ from . import index from .app import app, audio, completion, conversation, file, message, workflow -from .dataset import dataset, document, hit_testing, segment +from .dataset import dataset, document, hit_testing, segment, upload_file diff --git a/api/controllers/service_api/dataset/upload_file.py b/api/controllers/service_api/dataset/upload_file.py new file mode 100644 index 00000000000000..6382b63ea90269 --- /dev/null +++ b/api/controllers/service_api/dataset/upload_file.py @@ -0,0 +1,54 @@ +from werkzeug.exceptions import NotFound + +from controllers.service_api import api +from controllers.service_api.wraps import ( + DatasetApiResource, +) +from core.file import helpers as file_helpers +from extensions.ext_database import db +from models.dataset import Dataset +from models.model import UploadFile +from services.dataset_service import DocumentService + + +class UploadFileApi(DatasetApiResource): + def get(self, tenant_id, dataset_id, document_id): + """Get upload file.""" + # check dataset + dataset_id = str(dataset_id) + tenant_id = str(tenant_id) + dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first() + if not dataset: + raise NotFound("Dataset not found.") + # check document + document_id = str(document_id) + document = DocumentService.get_document(dataset.id, document_id) + if not document: + raise NotFound("Document not found.") + # check upload file + if document.data_source_type != "upload_file": + raise ValueError(f"Document data source type ({document.data_source_type}) is not upload_file.") + data_source_info = document.data_source_info_dict + if data_source_info and "upload_file_id" in data_source_info: + file_id = data_source_info["upload_file_id"] + upload_file = db.session.query(UploadFile).filter(UploadFile.id == file_id).first() + if not upload_file: + raise NotFound("UploadFile not found.") + else: + raise ValueError("Upload file id not found in document data source info.") + + url = file_helpers.get_signed_file_url(upload_file_id=upload_file.id) + return { + "id": upload_file.id, + "name": upload_file.name, + "size": upload_file.size, + "extension": upload_file.extension, + "url": url, + "download_url": f"{url}&as_attachment=true", + "mime_type": upload_file.mime_type, + "created_by": upload_file.created_by, + "created_at": upload_file.created_at.timestamp(), + }, 200 + + +api.add_resource(UploadFileApi, "/datasets//documents//upload-file") diff --git a/web/app/(commonLayout)/datasets/template/template.en.mdx b/web/app/(commonLayout)/datasets/template/template.en.mdx index 81b7842da4ac2f..b38cf38b9a6dfb 100644 --- a/web/app/(commonLayout)/datasets/template/template.en.mdx +++ b/web/app/(commonLayout)/datasets/template/template.en.mdx @@ -1106,6 +1106,57 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from
+ + + + ### Path + + + Knowledge ID + + + Document ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/upload-file' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' + ``` + + + ```json {{ title: 'Response' }} + { + "id": "file_id", + "name": "file_name", + "size": 1024, + "extension": "txt", + "url": "preview_url", + "download_url": "download_url", + "mime_type": "text/plain", + "created_by": "user_id", + "created_at": 1728734540, + } + ``` + + + + +
+ + + + + ### Path + + + 知识库 ID + + + 文档 ID + + + + + + ```bash {{ title: 'cURL' }} + curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/upload-file' \ + --header 'Authorization: Bearer {api_key}' \ + --header 'Content-Type: application/json' + ``` + + + ```json {{ title: 'Response' }} + { + "id": "file_id", + "name": "file_name", + "size": 1024, + "extension": "txt", + "url": "preview_url", + "download_url": "download_url", + "mime_type": "text/plain", + "created_by": "user_id", + "created_at": 1728734540, + } + ``` + + + + +
+ Date: Thu, 9 Jan 2025 15:16:41 +0800 Subject: [PATCH 21/42] fix: same chunk insert deadlock (#12502) Co-authored-by: huangzhuo --- api/core/indexing_runner.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/api/core/indexing_runner.py b/api/core/indexing_runner.py index 05000c5400ff9e..1bc4baf9c0e9e7 100644 --- a/api/core/indexing_runner.py +++ b/api/core/indexing_runner.py @@ -530,7 +530,6 @@ def _load( # chunk nodes by chunk size indexing_start_at = time.perf_counter() tokens = 0 - chunk_size = 10 if dataset_document.doc_form != IndexType.PARENT_CHILD_INDEX: # create keyword index create_keyword_thread = threading.Thread( @@ -539,11 +538,22 @@ def _load( ) create_keyword_thread.start() + max_workers = 10 if dataset.indexing_technique == "high_quality": - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: futures = [] - for i in range(0, len(documents), chunk_size): - chunk_documents = documents[i : i + chunk_size] + + # Distribute documents into multiple groups based on the hash values of page_content + # This is done to prevent multiple threads from processing the same document, + # Thereby avoiding potential database insertion deadlocks + document_groups: list[list[Document]] = [[] for _ in range(max_workers)] + for document in documents: + hash = helper.generate_text_hash(document.page_content) + group_index = int(hash, 16) % max_workers + document_groups[group_index].append(document) + for chunk_documents in document_groups: + if len(chunk_documents) == 0: + continue futures.append( executor.submit( self._process_chunk, From f230a9232e00f0c4288ba3f1959d84fe06cc35e3 Mon Sep 17 00:00:00 2001 From: lotsik Date: Thu, 9 Jan 2025 10:30:43 +0300 Subject: [PATCH 22/42] fix: Parsing OpenAPI spec for external tools (#12518) (#12530) --- api/core/tools/utils/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/core/tools/utils/parser.py b/api/core/tools/utils/parser.py index f1dc1123b9935f..30e4fdcf06dbc4 100644 --- a/api/core/tools/utils/parser.py +++ b/api/core/tools/utils/parser.py @@ -112,7 +112,7 @@ def parse_openapi_to_tool_bundle( llm_description=property.get("description", ""), default=property.get("default", None), placeholder=I18nObject( - en_US=parameter.get("description", ""), zh_Hans=parameter.get("description", "") + en_US=property.get("description", ""), zh_Hans=property.get("description", "") ), ) From a085ad47192b5a76f2d5587e226cf13d3f8a35bc Mon Sep 17 00:00:00 2001 From: crazywoola <100913391+crazywoola@users.noreply.github.com> Date: Thu, 9 Jan 2025 15:36:13 +0800 Subject: [PATCH 23/42] feat: show workflow running status (#12531) --- web/app/components/app/text-generate/item/index.tsx | 10 ++++++++-- web/app/components/base/chat/chat/answer/index.tsx | 9 +++++---- .../base/chat/chat/answer/workflow-process.tsx | 10 ++++++---- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/web/app/components/app/text-generate/item/index.tsx b/web/app/components/app/text-generate/item/index.tsx index ac868e6ee3c745..3e2f837685e4fb 100644 --- a/web/app/components/app/text-generate/item/index.tsx +++ b/web/app/components/app/text-generate/item/index.tsx @@ -306,8 +306,14 @@ const GenerationItem: FC = ({ }
- {siteInfo && siteInfo.show_workflow_steps && workflowProcessData && ( - + {siteInfo && workflowProcessData && ( + )} {workflowProcessData && !isError && ( diff --git a/web/app/components/base/chat/chat/answer/index.tsx b/web/app/components/base/chat/chat/answer/index.tsx index c6d14ddeade9e1..2ceaf81e785257 100644 --- a/web/app/components/base/chat/chat/answer/index.tsx +++ b/web/app/components/base/chat/chat/answer/index.tsx @@ -13,7 +13,7 @@ import AgentContent from './agent-content' import BasicContent from './basic-content' import SuggestedQuestions from './suggested-questions' import More from './more' -import WorkflowProcess from './workflow-process' +import WorkflowProcessItem from './workflow-process' import LoadingAnim from '@/app/components/base/chat/chat/loading-anim' import Citation from '@/app/components/base/chat/chat/citation' import { EditTitle } from '@/app/components/app/annotation/edit-annotation-modal/edit-item' @@ -133,7 +133,7 @@ const Answer: FC = ({ {/** Render the normal steps */} { workflowProcess && !hideProcessDetail && ( - = ({ } {/** Hide workflow steps by it's settings in siteInfo */} { - workflowProcess && hideProcessDetail && appData && appData.site.show_workflow_steps && ( - ) } diff --git a/web/app/components/base/chat/chat/answer/workflow-process.tsx b/web/app/components/base/chat/chat/answer/workflow-process.tsx index bb9abdb6fcf9b9..4dcac1aafc7eed 100644 --- a/web/app/components/base/chat/chat/answer/workflow-process.tsx +++ b/web/app/components/base/chat/chat/answer/workflow-process.tsx @@ -23,6 +23,7 @@ type WorkflowProcessProps = { expand?: boolean hideInfo?: boolean hideProcessDetail?: boolean + readonly?: boolean } const WorkflowProcessItem = ({ data, @@ -30,6 +31,7 @@ const WorkflowProcessItem = ({ expand = false, hideInfo = false, hideProcessDetail = false, + readonly = false, }: WorkflowProcessProps) => { const { t } = useTranslation() const [collapse, setCollapse] = useState(!expand) @@ -81,8 +83,8 @@ const WorkflowProcessItem = ({ }} >
setCollapse(!collapse)} + className={cn('flex items-center cursor-pointer', !collapse && 'px-1.5', readonly && 'cursor-default')} + onClick={() => !readonly && setCollapse(!collapse)} > { running && ( @@ -102,10 +104,10 @@ const WorkflowProcessItem = ({
{t('workflow.common.workflowProcess')}
- + {!readonly && }
{ - !collapse && ( + !collapse && !readonly && (
{ Date: Thu, 9 Jan 2025 16:04:14 +0800 Subject: [PATCH 24/42] fix: sum costs return error value on overview page (#12534) --- web/app/components/app/overview/appChart.tsx | 7 +++---- web/package.json | 1 + web/yarn.lock | 5 +++++ 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/web/app/components/app/overview/appChart.tsx b/web/app/components/app/overview/appChart.tsx index e0788bcda3bed6..43b1cb6afe5b98 100644 --- a/web/app/components/app/overview/appChart.tsx +++ b/web/app/components/app/overview/appChart.tsx @@ -6,6 +6,7 @@ import type { EChartsOption } from 'echarts' import useSWR from 'swr' import dayjs from 'dayjs' import { get } from 'lodash-es' +import Decimal from 'decimal.js' import { useTranslation } from 'react-i18next' import { formatNumber } from '@/utils/format' import Basic from '@/app/components/app-sidebar/basic' @@ -60,10 +61,8 @@ const CHART_TYPE_CONFIG: Record = { }, } -const sum = (arr: number[]): number => { - return arr.reduce((acr, cur) => { - return acr + cur - }) +const sum = (arr: Decimal.Value[]): number => { + return Decimal.sum(...arr).toNumber() } const defaultPeriod = { diff --git a/web/package.json b/web/package.json index 304a42871bd14d..7afb766d87b21d 100644 --- a/web/package.json +++ b/web/package.json @@ -50,6 +50,7 @@ "copy-to-clipboard": "^3.3.3", "crypto-js": "^4.2.0", "dayjs": "^1.11.7", + "decimal.js": "^10.4.3", "echarts": "^5.5.1", "echarts-for-react": "^3.0.2", "elkjs": "^0.9.3", diff --git a/web/yarn.lock b/web/yarn.lock index 339f47c236e054..6eed53dd39fdcd 100644 --- a/web/yarn.lock +++ b/web/yarn.lock @@ -5568,6 +5568,11 @@ decimal.js@^10.4.2: resolved "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz" integrity sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA== +decimal.js@^10.4.3: + version "10.4.3" + resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.4.3.tgz#1044092884d245d1b7f65725fa4ad4c6f781cc23" + integrity sha512-VBBaLc1MgL5XpzgIP7ny5Z6Nx3UrRkIViUkPUdtl9aya5amy3De1gsUUSB1g3+3sExYNjCAsAznmukyxCb1GRA== + decode-named-character-reference@^1.0.0: version "1.0.2" resolved "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz" From 2e97ba5700d7a166373805cf4307ca2477f55a1b Mon Sep 17 00:00:00 2001 From: Wu Tianwei <30284043+WTW0313@users.noreply.github.com> Date: Thu, 9 Jan 2025 17:44:11 +0800 Subject: [PATCH 25/42] fix: Add datasets list access control and fix datasets config display issue (#12533) Co-authored-by: nite-knite --- web/app/(commonLayout)/datasets/Container.tsx | 21 +++++-- web/app/(commonLayout)/datasets/Datasets.tsx | 10 +++- .../datasets/template/template.en.mdx | 48 +++++++++++++++- .../datasets/template/template.zh.mdx | 50 +++++++++++++++-- .../datasets/create/step-two/index.tsx | 14 +++-- .../datasets/settings/form/index.tsx | 56 ++++++++++--------- web/app/components/develop/md.tsx | 7 +++ web/i18n/en-US/dataset.ts | 2 + web/i18n/zh-Hans/dataset.ts | 2 + web/models/datasets.ts | 11 ++++ web/service/datasets.ts | 3 +- 11 files changed, 174 insertions(+), 50 deletions(-) diff --git a/web/app/(commonLayout)/datasets/Container.tsx b/web/app/(commonLayout)/datasets/Container.tsx index a0edb1cd61ba3c..c39d9c5dbfd69b 100644 --- a/web/app/(commonLayout)/datasets/Container.tsx +++ b/web/app/(commonLayout)/datasets/Container.tsx @@ -4,7 +4,8 @@ import { useEffect, useMemo, useRef, useState } from 'react' import { useRouter } from 'next/navigation' import { useTranslation } from 'react-i18next' -import { useDebounceFn } from 'ahooks' +import { useBoolean, useDebounceFn } from 'ahooks' +import { useQuery } from '@tanstack/react-query' // Components import ExternalAPIPanel from '../../components/datasets/external-api/external-api-panel' @@ -16,7 +17,9 @@ import TabSliderNew from '@/app/components/base/tab-slider-new' import TagManagementModal from '@/app/components/base/tag-management' import TagFilter from '@/app/components/base/tag-management/filter' import Button from '@/app/components/base/button' +import Input from '@/app/components/base/input' import { ApiConnectionMod } from '@/app/components/base/icons/src/vender/solid/development' +import CheckboxWithLabel from '@/app/components/datasets/create/website/base/checkbox-with-label' // Services import { fetchDatasetApiBaseUrl } from '@/service/datasets' @@ -26,16 +29,14 @@ import { useTabSearchParams } from '@/hooks/use-tab-searchparams' import { useStore as useTagStore } from '@/app/components/base/tag-management/store' import { useAppContext } from '@/context/app-context' import { useExternalApiPanel } from '@/context/external-api-panel-context' -// eslint-disable-next-line import/order -import { useQuery } from '@tanstack/react-query' -import Input from '@/app/components/base/input' const Container = () => { const { t } = useTranslation() const router = useRouter() - const { currentWorkspace } = useAppContext() + const { currentWorkspace, isCurrentWorkspaceOwner } = useAppContext() const showTagManagementModal = useTagStore(s => s.showTagManagementModal) const { showExternalApiPanel, setShowExternalApiPanel } = useExternalApiPanel() + const [includeAll, { toggle: toggleIncludeAll }] = useBoolean(false) const options = useMemo(() => { return [ @@ -90,6 +91,14 @@ const Container = () => { /> {activeTab === 'dataset' && (
+ {isCurrentWorkspaceOwner && } {
{activeTab === 'dataset' && ( <> - + {showTagManagementModal && ( diff --git a/web/app/(commonLayout)/datasets/Datasets.tsx b/web/app/(commonLayout)/datasets/Datasets.tsx index db6cb4a5185c57..ea918a2b17d763 100644 --- a/web/app/(commonLayout)/datasets/Datasets.tsx +++ b/web/app/(commonLayout)/datasets/Datasets.tsx @@ -6,7 +6,7 @@ import { debounce } from 'lodash-es' import { useTranslation } from 'react-i18next' import NewDatasetCard from './NewDatasetCard' import DatasetCard from './DatasetCard' -import type { DataSetListResponse } from '@/models/datasets' +import type { DataSetListResponse, FetchDatasetsParams } from '@/models/datasets' import { fetchDatasets } from '@/service/datasets' import { useAppContext } from '@/context/app-context' @@ -15,13 +15,15 @@ const getKey = ( previousPageData: DataSetListResponse, tags: string[], keyword: string, + includeAll: boolean, ) => { if (!pageIndex || previousPageData.has_more) { - const params: any = { + const params: FetchDatasetsParams = { url: 'datasets', params: { page: pageIndex + 1, limit: 30, + include_all: includeAll, }, } if (tags.length) @@ -37,16 +39,18 @@ type Props = { containerRef: React.RefObject tags: string[] keywords: string + includeAll: boolean } const Datasets = ({ containerRef, tags, keywords, + includeAll, }: Props) => { const { isCurrentWorkspaceEditor } = useAppContext() const { data, isLoading, setSize, mutate } = useSWRInfinite( - (pageIndex: number, previousPageData: DataSetListResponse) => getKey(pageIndex, previousPageData, tags, keywords), + (pageIndex: number, previousPageData: DataSetListResponse) => getKey(pageIndex, previousPageData, tags, keywords, includeAll), fetchDatasets, { revalidateFirstPage: false, revalidateAll: true }, ) diff --git a/web/app/(commonLayout)/datasets/template/template.en.mdx b/web/app/(commonLayout)/datasets/template/template.en.mdx index b38cf38b9a6dfb..3fa22a1620ed9e 100644 --- a/web/app/(commonLayout)/datasets/template/template.en.mdx +++ b/web/app/(commonLayout)/datasets/template/template.en.mdx @@ -1,5 +1,5 @@ import { CodeGroup } from '@/app/components/develop/code.tsx' -import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from '@/app/components/develop/md.tsx' +import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstruction, Paragraph } from '@/app/components/develop/md.tsx' # Knowledge API @@ -80,6 +80,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - max_tokens The maximum length (tokens) must be validated to be shorter than the length of the parent chunk - chunk_overlap Define the overlap between adjacent chunks (optional) + When no parameters are set for the knowledge base, the first upload requires the following parameters to be provided; if not provided, the default parameters will be used. + + Retrieval model + - search_method (string) Search method + - hybrid_search Hybrid search + - semantic_search Semantic search + - full_text_search Full-text search + - reranking_enable (bool) Whether to enable reranking + - reranking_mode (object) Rerank model configuration + - reranking_provider_name (string) Rerank model provider + - reranking_model_name (string) Rerank model name + - top_k (int) Number of results to return + - score_threshold_enabled (bool) Whether to enable score threshold + - score_threshold (float) Score threshold + + + Embedding model name + + + Embedding model provider + @@ -197,6 +218,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from Files that need to be uploaded. + When no parameters are set for the knowledge base, the first upload requires the following parameters to be provided; if not provided, the default parameters will be used. + + Retrieval model + - search_method (string) Search method + - hybrid_search Hybrid search + - semantic_search Semantic search + - full_text_search Full-text search + - reranking_enable (bool) Whether to enable reranking + - reranking_mode (object) Rerank model configuration + - reranking_provider_name (string) Rerank model provider + - reranking_model_name (string) Rerank model name + - top_k (int) Number of results to return + - score_threshold_enabled (bool) Whether to enable score threshold + - score_threshold (float) Score threshold + + + Embedding model name + + + Embedding model provider + @@ -1188,10 +1230,10 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - reranking_mode (object) Rerank model configuration, required if reranking is enabled - reranking_provider_name (string) Rerank model provider - reranking_model_name (string) Rerank model name - - weights (double) Semantic search weight setting in hybrid search mode + - weights (float) Semantic search weight setting in hybrid search mode - top_k (integer) Number of results to return (optional) - score_threshold_enabled (bool) Whether to enable score threshold - - score_threshold (double) Score threshold + - score_threshold (float) Score threshold Unused field diff --git a/web/app/(commonLayout)/datasets/template/template.zh.mdx b/web/app/(commonLayout)/datasets/template/template.zh.mdx index 7bb773eee917c9..334591743f931b 100644 --- a/web/app/(commonLayout)/datasets/template/template.zh.mdx +++ b/web/app/(commonLayout)/datasets/template/template.zh.mdx @@ -1,5 +1,5 @@ import { CodeGroup } from '@/app/components/develop/code.tsx' -import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from '@/app/components/develop/md.tsx' +import { Row, Col, Properties, Property, Heading, SubProperty, PropertyInstruction, Paragraph } from '@/app/components/develop/md.tsx' # 知识库 API @@ -80,6 +80,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - max_tokens 最大长度 (token) 需要校验小于父级的长度 - chunk_overlap 分段重叠指的是在对数据进行分段时，段与段之间存在一定的重叠部分（选填） + 当知识库未设置任何参数的时候，首次上传需要提供以下参数，未提供则使用默认选项： + + 检索模式 + - search_method (string) 检索方法 + - hybrid_search 混合检索 + - semantic_search 语义检索 + - full_text_search 全文检索 + - reranking_enable (bool) 是否开启rerank + - reranking_model (object) Rerank 模型配置 + - reranking_provider_name (string) Rerank 模型的提供商 + - reranking_model_name (string) Rerank 模型的名称 + - top_k (int) 召回条数 + - score_threshold_enabled (bool)是否开启召回分数限制 + - score_threshold (float) 召回分数限制 + + + Embedding 模型名称 + + + Embedding 模型供应商 + @@ -197,6 +218,27 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from 需要上传的文件。 + 当知识库未设置任何参数的时候，首次上传需要提供以下参数，未提供则使用默认选项： + + 检索模式 + - search_method (string) 检索方法 + - hybrid_search 混合检索 + - semantic_search 语义检索 + - full_text_search 全文检索 + - reranking_enable (bool) 是否开启rerank + - reranking_model (object) Rerank 模型配置 + - reranking_provider_name (string) Rerank 模型的提供商 + - reranking_model_name (string) Rerank 模型的名称 + - top_k (int) 召回条数 + - score_threshold_enabled (bool)是否开启召回分数限制 + - score_threshold (float) 召回分数限制 + + + Embedding 模型名称 + + + Embedding 模型供应商 + @@ -1186,13 +1228,13 @@ import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from - full_text_search 全文检索 - hybrid_search 混合检索 - reranking_enable (bool) 是否启用 Reranking，非必填，如果检索模式为 semantic_search 模式或者 hybrid_search 则传值 - - reranking_mode (object) Rerank模型配置，非必填，如果启用了 reranking 则传值 + - reranking_mode (object) Rerank 模型配置，非必填，如果启用了 reranking 则传值 - reranking_provider_name (string) Rerank 模型提供商 - reranking_model_name (string) Rerank 模型名称 - - weights (double) 混合检索模式下语意检索的权重设置 + - weights (float) 混合检索模式下语意检索的权重设置 - top_k (integer) 返回结果数量，非必填 - score_threshold_enabled (bool) 是否开启 score 阈值 - - score_threshold (double) Score 阈值 + - score_threshold (float) Score 阈值 未启用字段 diff --git a/web/app/components/datasets/create/step-two/index.tsx b/web/app/components/datasets/create/step-two/index.tsx index 27ca16579bcc56..11984d71c64987 100644 --- a/web/app/components/datasets/create/step-two/index.tsx +++ b/web/app/components/datasets/create/step-two/index.tsx @@ -575,6 +575,8 @@ const StepTwo = ({ const economyDomRef = useRef(null) const isHoveringEconomy = useHover(economyDomRef) + const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type + return (
@@ -931,15 +933,15 @@ const StepTwo = ({
{t('datasetSettings.form.embeddingModel')}
{ setEmbeddingModel(model) }} /> - {!!datasetId && ( + {isModelAndRetrievalConfigDisabled && (
{t('datasetCreation.stepTwo.indexSettingTip')} {t('datasetCreation.stepTwo.datasetSettingLink')} @@ -950,7 +952,7 @@ const StepTwo = ({ {/* Retrieval Method Config */}
- {!datasetId + {!isModelAndRetrievalConfigDisabled ? (
{t('datasetSettings.form.retrievalSetting.title')}
@@ -971,14 +973,14 @@ const StepTwo = ({ getIndexing_technique() === IndexingType.QUALIFIED ? ( ) : ( diff --git a/web/app/components/datasets/settings/form/index.tsx b/web/app/components/datasets/settings/form/index.tsx index 760954d6cbf4fa..42ea7d637b37c3 100644 --- a/web/app/components/datasets/settings/form/index.tsx +++ b/web/app/components/datasets/settings/form/index.tsx @@ -223,7 +223,7 @@ const Form = () => { setIndexMethod(v)} + onChange={v => setIndexMethod(v!)} docForm={currentDataset.doc_form} currentValue={currentDataset.indexing_technique} /> @@ -300,35 +300,37 @@ const Form = () => {
- : <> -
-
-
-
-
{t('datasetSettings.form.retrievalSetting.title')}
-
- {t('datasetSettings.form.retrievalSetting.learnMore')} - {t('datasetSettings.form.retrievalSetting.description')} + : indexMethod + ? <> +
+
+
+
+
{t('datasetSettings.form.retrievalSetting.title')}
+
+ {t('datasetSettings.form.retrievalSetting.learnMore')} + {t('datasetSettings.form.retrievalSetting.description')} +
+
+ {indexMethod === IndexingType.QUALIFIED + ? ( + + ) + : ( + + )} +
-
- {indexMethod === 'high_quality' - ? ( - - ) - : ( - - )} -
-
- + + : null }
diff --git a/web/app/components/develop/md.tsx b/web/app/components/develop/md.tsx index 26b4007c8704c9..c75798fcfe055a 100644 --- a/web/app/components/develop/md.tsx +++ b/web/app/components/develop/md.tsx @@ -1,4 +1,5 @@ 'use client' +import type { PropsWithChildren } from 'react' import classNames from '@/utils/classnames' type IChildrenProps = { @@ -139,3 +140,9 @@ export function SubProperty({ name, type, children }: ISubProperty) { ) } + +export function PropertyInstruction({ children }: PropsWithChildren<{}>) { + return ( +
  • {children}
    • + ) +} diff --git a/web/i18n/en-US/dataset.ts b/web/i18n/en-US/dataset.ts index 6a6df700d7b304..4e1f2549d8abd3 100644 --- a/web/i18n/en-US/dataset.ts +++ b/web/i18n/en-US/dataset.ts @@ -166,6 +166,8 @@ const translation = { cancel: 'Cancel', }, preprocessDocument: '{{num}} Preprocess Documents', + allKnowledge: 'All Knowledge', + allKnowledgeDescription: 'Select to display all knowledge in this workspace. Only the Workspace Owner can manage all knowledge.', } export default translation diff --git a/web/i18n/zh-Hans/dataset.ts b/web/i18n/zh-Hans/dataset.ts index d7834b41162ada..bedd114b7343c8 100644 --- a/web/i18n/zh-Hans/dataset.ts +++ b/web/i18n/zh-Hans/dataset.ts @@ -166,6 +166,8 @@ const translation = { cancel: '取消', }, preprocessDocument: '{{num}} 个预处理文档', + allKnowledge: '所有知识库', + allKnowledgeDescription: '选择以显示该工作区内所有知识库。只有工作区所有者才能管理所有知识库。', } export default translation diff --git a/web/models/datasets.ts b/web/models/datasets.ts index 9d4768b67c2c45..673fb5fb1530eb 100644 --- a/web/models/datasets.ts +++ b/web/models/datasets.ts @@ -132,6 +132,17 @@ export type FileItem = { progress: number } +export type FetchDatasetsParams = { + url: string + params: { + page: number + tag_ids?: string[] + limit: number + include_all: boolean + keyword?: string + } +} + export type DataSetListResponse = { data: DataSet[] has_more: boolean diff --git a/web/service/datasets.ts b/web/service/datasets.ts index 87f4e3a63858de..f2065de382eb91 100644 --- a/web/service/datasets.ts +++ b/web/service/datasets.ts @@ -13,6 +13,7 @@ import type { ExternalAPIUsage, ExternalKnowledgeBaseHitTestingResponse, ExternalKnowledgeItem, + FetchDatasetsParams, FileIndexingEstimateResponse, HitTestingRecordsResponse, HitTestingResponse, @@ -67,7 +68,7 @@ export const fetchDatasetRelatedApps: Fetcher = (dat return get(`/datasets/${datasetId}/related-apps`) } -export const fetchDatasets: Fetcher = ({ url, params }) => { +export const fetchDatasets: Fetcher = ({ url, params }) => { const urlParams = qs.stringify(params, { indices: false }) return get(`${url}?${urlParams}`) } From 14ee51aeadee5ecf45a5ac6b99be248f2e76bd30 Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:21:25 +0800 Subject: [PATCH 26/42] Feat/add knowledge include all filter (#12537) --- api/controllers/console/datasets/datasets.py | 4 ++-- api/controllers/service_api/dataset/dataset.py | 5 ++++- api/services/dataset_service.py | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 2da45a7bb67bc6..386e45c58e5042 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -52,12 +52,12 @@ def get(self): # provider = request.args.get("provider", default="vendor") search = request.args.get("keyword", default=None, type=str) tag_ids = request.args.getlist("tag_ids") - + include_all = request.args.get("include_all", default="false").lower() == "true" if ids: datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id) else: datasets, total = DatasetService.get_datasets( - page, limit, current_user.current_tenant_id, current_user, search, tag_ids + page, limit, current_user.current_tenant_id, current_user, search, tag_ids, include_all ) # check embedding setting diff --git a/api/controllers/service_api/dataset/dataset.py b/api/controllers/service_api/dataset/dataset.py index d6a3beb6b80b9d..49acdd693a8431 100644 --- a/api/controllers/service_api/dataset/dataset.py +++ b/api/controllers/service_api/dataset/dataset.py @@ -31,8 +31,11 @@ def get(self, tenant_id): # provider = request.args.get("provider", default="vendor") search = request.args.get("keyword", default=None, type=str) tag_ids = request.args.getlist("tag_ids") + include_all = request.args.get("include_all", default="false").lower() == "true" - datasets, total = DatasetService.get_datasets(page, limit, tenant_id, current_user, search, tag_ids) + datasets, total = DatasetService.get_datasets( + page, limit, tenant_id, current_user, search, tag_ids, include_all + ) # check embedding setting provider_manager = ProviderManager() configurations = provider_manager.get_configurations(tenant_id=current_user.current_tenant_id) diff --git a/api/services/dataset_service.py b/api/services/dataset_service.py index 82433b64ffcc3a..dac0a6a7728fe0 100644 --- a/api/services/dataset_service.py +++ b/api/services/dataset_service.py @@ -71,7 +71,7 @@ class DatasetService: @staticmethod - def get_datasets(page, per_page, tenant_id=None, user=None, search=None, tag_ids=None): + def get_datasets(page, per_page, tenant_id=None, user=None, search=None, tag_ids=None, include_all=False): query = Dataset.query.filter(Dataset.tenant_id == tenant_id).order_by(Dataset.created_at.desc()) if user: @@ -86,7 +86,7 @@ def get_datasets(page, per_page, tenant_id=None, user=None, search=None, tag_ids else: return [], 0 else: - if user.current_role != TenantAccountRole.OWNER: + if user.current_role != TenantAccountRole.OWNER or not include_all: # show all datasets that the user has permission to access if permitted_dataset_ids: query = query.filter( From 140965b738fd5eec13b4f92caf5e84e86c6c45a3 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:30:06 +0800 Subject: [PATCH 27/42] chore: translate i18n files (#12543) Co-authored-by: WTW0313 <30284043+WTW0313@users.noreply.github.com> --- web/i18n/de-DE/common.ts | 1 + web/i18n/de-DE/dataset.ts | 2 ++ web/i18n/es-ES/common.ts | 1 + web/i18n/es-ES/dataset.ts | 2 ++ web/i18n/fa-IR/common.ts | 1 + web/i18n/fa-IR/dataset.ts | 2 ++ web/i18n/fr-FR/common.ts | 1 + web/i18n/fr-FR/dataset.ts | 2 ++ web/i18n/hi-IN/common.ts | 1 + web/i18n/hi-IN/dataset.ts | 2 ++ web/i18n/it-IT/common.ts | 1 + web/i18n/it-IT/dataset.ts | 2 ++ web/i18n/ja-JP/common.ts | 1 + web/i18n/ja-JP/dataset.ts | 2 ++ web/i18n/ko-KR/common.ts | 1 + web/i18n/ko-KR/dataset.ts | 2 ++ web/i18n/pl-PL/common.ts | 1 + web/i18n/pl-PL/dataset.ts | 2 ++ web/i18n/pt-BR/common.ts | 1 + web/i18n/pt-BR/dataset.ts | 2 ++ web/i18n/ro-RO/common.ts | 1 + web/i18n/ro-RO/dataset.ts | 2 ++ web/i18n/ru-RU/common.ts | 1 + web/i18n/ru-RU/dataset.ts | 2 ++ web/i18n/sl-SI/common.ts | 1 + web/i18n/sl-SI/dataset.ts | 2 ++ web/i18n/th-TH/common.ts | 1 + web/i18n/th-TH/dataset.ts | 2 ++ web/i18n/tr-TR/common.ts | 1 + web/i18n/tr-TR/dataset.ts | 2 ++ web/i18n/uk-UA/common.ts | 1 + web/i18n/uk-UA/dataset.ts | 2 ++ web/i18n/vi-VN/common.ts | 1 + web/i18n/vi-VN/dataset.ts | 2 ++ web/i18n/zh-Hant/common.ts | 1 + web/i18n/zh-Hant/dataset.ts | 2 ++ 36 files changed, 54 insertions(+) diff --git a/web/i18n/de-DE/common.ts b/web/i18n/de-DE/common.ts index f97403ed5e45f7..915fc2a27721a5 100644 --- a/web/i18n/de-DE/common.ts +++ b/web/i18n/de-DE/common.ts @@ -49,6 +49,7 @@ const translation = { view: 'Ansehen', submit: 'Senden', skip: 'Schiff', + imageCopied: 'Kopiertes Bild', }, placeholder: { input: 'Bitte eingeben', diff --git a/web/i18n/de-DE/dataset.ts b/web/i18n/de-DE/dataset.ts index e0bc91723c6a5e..3d3535b32c435f 100644 --- a/web/i18n/de-DE/dataset.ts +++ b/web/i18n/de-DE/dataset.ts @@ -166,6 +166,8 @@ const translation = { localDocs: 'Lokale Dokumente', preprocessDocument: '{{num}} Vorverarbeiten von Dokumenten', documentsDisabled: '{{num}} Dokumente deaktiviert - seit über 30 Tagen inaktiv', + allKnowledge: 'Alles Wissen', + allKnowledgeDescription: 'Wählen Sie diese Option aus, um das gesamte Wissen in diesem Arbeitsbereich anzuzeigen. Nur der Workspace-Besitzer kann das gesamte Wissen verwalten.', } export default translation diff --git a/web/i18n/es-ES/common.ts b/web/i18n/es-ES/common.ts index 074f9385c2bd03..936da902003b33 100644 --- a/web/i18n/es-ES/common.ts +++ b/web/i18n/es-ES/common.ts @@ -49,6 +49,7 @@ const translation = { view: 'Vista', submit: 'Enviar', skip: 'Navío', + imageCopied: 'Imagen copiada', }, errorMsg: { fieldRequired: '{{field}} es requerido', diff --git a/web/i18n/es-ES/dataset.ts b/web/i18n/es-ES/dataset.ts index 5fb668f1f3b2e4..3bfdd8f46c0a00 100644 --- a/web/i18n/es-ES/dataset.ts +++ b/web/i18n/es-ES/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '{{num}} Documentos desactivados - inactivos durante más de 30 días', preprocessDocument: '{{num}} Documentos de preprocesamiento', localDocs: 'Documentos locales', + allKnowledgeDescription: 'Seleccione esta opción para mostrar todos los conocimientos de este espacio de trabajo. Solo el propietario del espacio de trabajo puede administrar todo el conocimiento.', + allKnowledge: 'Todo el conocimiento', } export default translation diff --git a/web/i18n/fa-IR/common.ts b/web/i18n/fa-IR/common.ts index aed31524650534..53086071aae13f 100644 --- a/web/i18n/fa-IR/common.ts +++ b/web/i18n/fa-IR/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: 'ذخیره و بازسازی تکه های فرزند', submit: 'ارسال', skip: 'کشتی', + imageCopied: 'تصویر کپی شده', }, errorMsg: { fieldRequired: '{{field}} الزامی است', diff --git a/web/i18n/fa-IR/dataset.ts b/web/i18n/fa-IR/dataset.ts index c8cc83ae9c041b..70012a05904ce0 100644 --- a/web/i18n/fa-IR/dataset.ts +++ b/web/i18n/fa-IR/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '{{num}} اسناد غیرفعال - غیرفعال برای بیش از 30 روز', preprocessDocument: '{{عدد}} اسناد پیش پردازش', localDocs: 'اسناد محلی', + allKnowledge: 'همه دانش ها', + allKnowledgeDescription: 'برای نمایش تمام دانش در این فضای کاری انتخاب کنید. فقط مالک فضای کاری می تواند تمام دانش را مدیریت کند.', } export default translation diff --git a/web/i18n/fr-FR/common.ts b/web/i18n/fr-FR/common.ts index 38aeafebab656c..662f53ab66ee74 100644 --- a/web/i18n/fr-FR/common.ts +++ b/web/i18n/fr-FR/common.ts @@ -49,6 +49,7 @@ const translation = { regenerate: 'Régénérer', submit: 'Envoyer', skip: 'Bateau', + imageCopied: 'Image copied', }, placeholder: { input: 'Veuillez entrer', diff --git a/web/i18n/fr-FR/dataset.ts b/web/i18n/fr-FR/dataset.ts index bdaea09eec5e28..b288e513efde75 100644 --- a/web/i18n/fr-FR/dataset.ts +++ b/web/i18n/fr-FR/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '{{num}} documents désactivés - inactifs depuis plus de 30 jours', localDocs: 'Docs locaux', enable: 'Activer', + allKnowledge: 'Toutes les connaissances', + allKnowledgeDescription: 'Sélectionnez cette option pour afficher toutes les connaissances dans cet espace de travail. Seul le propriétaire de l’espace de travail peut gérer toutes les connaissances.', } export default translation diff --git a/web/i18n/hi-IN/common.ts b/web/i18n/hi-IN/common.ts index 109a295fa1a980..2a14cd02798ac7 100644 --- a/web/i18n/hi-IN/common.ts +++ b/web/i18n/hi-IN/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: 'सहेजें और पुन: उत्पन्न करें बाल विखंडू', skip: 'जहाज़', submit: 'जमा करें', + imageCopied: 'कॉपी की गई छवि', }, errorMsg: { fieldRequired: '{{field}} आवश्यक है', diff --git a/web/i18n/hi-IN/dataset.ts b/web/i18n/hi-IN/dataset.ts index d3838e3dd0b6b1..b95f13088c8706 100644 --- a/web/i18n/hi-IN/dataset.ts +++ b/web/i18n/hi-IN/dataset.ts @@ -173,6 +173,8 @@ const translation = { preprocessDocument: '{{num}} प्रीप्रोसेस दस्तावेज़', enable: 'योग्य बनाना', documentsDisabled: '{{num}} दस्तावेज़ अक्षम - 30 दिनों से अधिक समय से निष्क्रिय', + allKnowledge: 'सर्व ज्ञान', + allKnowledgeDescription: 'इस कार्यस्थान में सभी ज्ञान प्रदर्शित करने के लिए चयन करें. केवल कार्यस्थान स्वामी ही सभी ज्ञान का प्रबंधन कर सकता है.', } export default translation diff --git a/web/i18n/it-IT/common.ts b/web/i18n/it-IT/common.ts index 9f5ec24dc3167c..c764a93dac908d 100644 --- a/web/i18n/it-IT/common.ts +++ b/web/i18n/it-IT/common.ts @@ -49,6 +49,7 @@ const translation = { viewMore: 'SCOPRI DI PIÙ', submit: 'Invia', skip: 'Nave', + imageCopied: 'Immagine copiata', }, errorMsg: { fieldRequired: '{{field}} è obbligatorio', diff --git a/web/i18n/it-IT/dataset.ts b/web/i18n/it-IT/dataset.ts index fc15ca5929b51c..dec41bec420531 100644 --- a/web/i18n/it-IT/dataset.ts +++ b/web/i18n/it-IT/dataset.ts @@ -173,6 +173,8 @@ const translation = { enable: 'Abilitare', documentsDisabled: '{{num}} documenti disabilitati - inattivi da oltre 30 giorni', localDocs: 'Documenti locali', + allKnowledge: 'Tutta la conoscenza', + allKnowledgeDescription: 'Selezionare questa opzione per visualizzare tutte le informazioni in questa area di lavoro. Solo il proprietario dell\'area di lavoro può gestire tutte le conoscenze.', } export default translation diff --git a/web/i18n/ja-JP/common.ts b/web/i18n/ja-JP/common.ts index 8944bda2fb1f7f..e176f5d139ba7c 100644 --- a/web/i18n/ja-JP/common.ts +++ b/web/i18n/ja-JP/common.ts @@ -49,6 +49,7 @@ const translation = { regenerate: '再生成', submit: '送信', skip: 'スキップ', + imageCopied: 'コピーした画像', }, errorMsg: { fieldRequired: '{{field}}は必要です', diff --git a/web/i18n/ja-JP/dataset.ts b/web/i18n/ja-JP/dataset.ts index f7a6968c10a945..f589d97b75507e 100644 --- a/web/i18n/ja-JP/dataset.ts +++ b/web/i18n/ja-JP/dataset.ts @@ -166,6 +166,8 @@ const translation = { cancel: 'キャンセル', }, preprocessDocument: '{{num}}件のドキュメントを前処理', + allKnowledge: 'すべての知識', + allKnowledgeDescription: 'このワークスペースにすべてのナレッジを表示する場合に選択します。ワークスペースのオーナーのみがすべてのナレッジを管理できます。', } export default translation diff --git a/web/i18n/ko-KR/common.ts b/web/i18n/ko-KR/common.ts index 207ffa44ccc409..a11af1ba31a505 100644 --- a/web/i18n/ko-KR/common.ts +++ b/web/i18n/ko-KR/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: '저장 및 자식 청크 재생성', submit: '전송', skip: '배', + imageCopied: '복사된 이미지', }, placeholder: { input: '입력해주세요', diff --git a/web/i18n/ko-KR/dataset.ts b/web/i18n/ko-KR/dataset.ts index db187ec421424f..4d622cf7f24c39 100644 --- a/web/i18n/ko-KR/dataset.ts +++ b/web/i18n/ko-KR/dataset.ts @@ -165,6 +165,8 @@ const translation = { preprocessDocument: '{{숫자}} 문서 전처리', enable: '사용', documentsDisabled: '{{num}} 문서 사용 안 함 - 30일 이상 비활성 상태', + allKnowledge: '모든 지식', + allKnowledgeDescription: '이 작업 영역의 모든 정보를 표시하려면 선택합니다. 워크스페이스 소유자만 모든 기술 자료를 관리할 수 있습니다.', } export default translation diff --git a/web/i18n/pl-PL/common.ts b/web/i18n/pl-PL/common.ts index 87e3dfe48ee280..d6502416c35a82 100644 --- a/web/i18n/pl-PL/common.ts +++ b/web/i18n/pl-PL/common.ts @@ -49,6 +49,7 @@ const translation = { close: 'Zamykać', submit: 'Prześlij', skip: 'Statek', + imageCopied: 'Skopiowany obraz', }, placeholder: { input: 'Proszę wprowadzić', diff --git a/web/i18n/pl-PL/dataset.ts b/web/i18n/pl-PL/dataset.ts index 3db197bd76d3a1..9a5ed10a5ab8f4 100644 --- a/web/i18n/pl-PL/dataset.ts +++ b/web/i18n/pl-PL/dataset.ts @@ -172,6 +172,8 @@ const translation = { localDocs: 'Lokalne dokumenty', documentsDisabled: '{{num}} dokumenty wyłączone - nieaktywne przez ponad 30 dni', enable: 'Umożliwiać', + allKnowledge: 'Cała wiedza', + allKnowledgeDescription: 'Wybierz tę opcję, aby wyświetlić całą wiedzę w tym obszarze roboczym. Tylko właściciel obszaru roboczego może zarządzać całą wiedzą.', } export default translation diff --git a/web/i18n/pt-BR/common.ts b/web/i18n/pt-BR/common.ts index ec1203a6485837..d0327de642a7ca 100644 --- a/web/i18n/pt-BR/common.ts +++ b/web/i18n/pt-BR/common.ts @@ -49,6 +49,7 @@ const translation = { view: 'Vista', submit: 'Enviar', skip: 'Navio', + imageCopied: 'Imagem copiada', }, placeholder: { input: 'Por favor, insira', diff --git a/web/i18n/pt-BR/dataset.ts b/web/i18n/pt-BR/dataset.ts index c3c3e3b631f027..c8214e1645b60a 100644 --- a/web/i18n/pt-BR/dataset.ts +++ b/web/i18n/pt-BR/dataset.ts @@ -166,6 +166,8 @@ const translation = { enable: 'Habilitar', preprocessDocument: '{{num}} Documentos de pré-processamento', localDocs: 'Documentos locais', + allKnowledgeDescription: 'Selecione para exibir todo o conhecimento neste espaço de trabalho. Somente o proprietário do espaço de trabalho pode gerenciar todo o conhecimento.', + allKnowledge: 'Todo o conhecimento', } export default translation diff --git a/web/i18n/ro-RO/common.ts b/web/i18n/ro-RO/common.ts index 39ecc896c688ed..8f0cbc64cbd9a9 100644 --- a/web/i18n/ro-RO/common.ts +++ b/web/i18n/ro-RO/common.ts @@ -49,6 +49,7 @@ const translation = { view: 'Vedere', submit: 'Prezinte', skip: 'Navă', + imageCopied: 'Imagine copiată', }, placeholder: { input: 'Vă rugăm să introduceți', diff --git a/web/i18n/ro-RO/dataset.ts b/web/i18n/ro-RO/dataset.ts index b73230f4c5281c..2feff675963f03 100644 --- a/web/i18n/ro-RO/dataset.ts +++ b/web/i18n/ro-RO/dataset.ts @@ -166,6 +166,8 @@ const translation = { preprocessDocument: '{{num}} Procesarea prealabilă a documentelor', enable: 'Activa', localDocs: 'Documente locale', + allKnowledge: 'Toate cunoștințele', + allKnowledgeDescription: 'Selectați pentru a afișa toate cunoștințele din acest spațiu de lucru. Doar proprietarul spațiului de lucru poate gestiona toate cunoștințele.', } export default translation diff --git a/web/i18n/ru-RU/common.ts b/web/i18n/ru-RU/common.ts index 4c606a1f65165c..2d8535e6a027ad 100644 --- a/web/i18n/ru-RU/common.ts +++ b/web/i18n/ru-RU/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: 'Сохранение и повторное создание дочерних блоков', submit: 'Отправить', skip: 'Корабль', + imageCopied: 'Скопированное изображение', }, errorMsg: { fieldRequired: '{{field}} обязательно', diff --git a/web/i18n/ru-RU/dataset.ts b/web/i18n/ru-RU/dataset.ts index c2831756f1acf7..41da4333f472d9 100644 --- a/web/i18n/ru-RU/dataset.ts +++ b/web/i18n/ru-RU/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: 'Документы {{num}} отключены - неактивны более 30 дней', localDocs: 'Местная документация', enable: 'Давать возможность', + allKnowledge: 'Все знания', + allKnowledgeDescription: 'Выберите, чтобы отобразить все знания в этой рабочей области. Только владелец рабочего пространства может управлять всеми знаниями.', } export default translation diff --git a/web/i18n/sl-SI/common.ts b/web/i18n/sl-SI/common.ts index cde5bf034e81fb..2d30e0b76cc2a4 100644 --- a/web/i18n/sl-SI/common.ts +++ b/web/i18n/sl-SI/common.ts @@ -49,6 +49,7 @@ const translation = { viewMore: 'POGLEJ VEČ', submit: 'Predložiti', skip: 'Ladja', + imageCopied: 'Kopirana slika', }, errorMsg: { fieldRequired: '{{field}} je obvezno', diff --git a/web/i18n/sl-SI/dataset.ts b/web/i18n/sl-SI/dataset.ts index e0d46be82f4eaa..0161a0ad8d420f 100644 --- a/web/i18n/sl-SI/dataset.ts +++ b/web/i18n/sl-SI/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '{{num}} dokumenti onemogočeni - neaktivni več kot 30 dni', preprocessDocument: '{{num}} Predobdelava dokumentov', enable: 'Omogočiti', + allKnowledge: 'Vse znanje', + allKnowledgeDescription: 'Izberite, če želite prikazati vse znanje v tem delovnem prostoru. Samo lastnik delovnega prostora lahko upravlja vse znanje.', } export default translation diff --git a/web/i18n/th-TH/common.ts b/web/i18n/th-TH/common.ts index ea8d5e5b1d10a2..ab6def82c5a59a 100644 --- a/web/i18n/th-TH/common.ts +++ b/web/i18n/th-TH/common.ts @@ -49,6 +49,7 @@ const translation = { close: 'ปิด', skip: 'เรือ', submit: 'ส่ง', + imageCopied: 'ภาพที่คัดลอก', }, errorMsg: { fieldRequired: '{{field}} เป็นสิ่งจําเป็น', diff --git a/web/i18n/th-TH/dataset.ts b/web/i18n/th-TH/dataset.ts index a2a07bf5a11e2f..1877226dc42fed 100644 --- a/web/i18n/th-TH/dataset.ts +++ b/web/i18n/th-TH/dataset.ts @@ -165,6 +165,8 @@ const translation = { preprocessDocument: '{{num}} เอกสารการประมวลผลล่วงหน้า', documentsDisabled: '{{num}} เอกสารถูกปิดใช้งาน - ไม่ได้ใช้งานนานกว่า 30 วัน', enable: 'เปิด', + allKnowledge: 'ความรู้ทั้งหมด', + allKnowledgeDescription: 'เลือกเพื่อแสดงความรู้ทั้งหมดในพื้นที่ทํางานนี้ เฉพาะเจ้าของพื้นที่ทํางานเท่านั้นที่สามารถจัดการความรู้ทั้งหมดได้', } export default translation diff --git a/web/i18n/tr-TR/common.ts b/web/i18n/tr-TR/common.ts index a427537a7bbe7f..ea764ebd299232 100644 --- a/web/i18n/tr-TR/common.ts +++ b/web/i18n/tr-TR/common.ts @@ -49,6 +49,7 @@ const translation = { close: 'Kapatmak', submit: 'Gönder', skip: 'Gemi', + imageCopied: 'Kopyalanan görüntü', }, errorMsg: { fieldRequired: '{{field}} gereklidir', diff --git a/web/i18n/tr-TR/dataset.ts b/web/i18n/tr-TR/dataset.ts index facaf3ee5b6ef3..6183849ebc384f 100644 --- a/web/i18n/tr-TR/dataset.ts +++ b/web/i18n/tr-TR/dataset.ts @@ -166,6 +166,8 @@ const translation = { localDocs: 'Yerel Dokümanlar', documentsDisabled: '{{num}} belge devre dışı - 30 günden uzun süre etkin değil', enable: 'Etkinleştirmek', + allKnowledge: 'Tüm Bilgiler', + allKnowledgeDescription: 'Bu çalışma alanındaki tüm bilgileri görüntülemek için seçin. Yalnızca Çalışma Alanı Sahibi tüm bilgileri yönetebilir.', } export default translation diff --git a/web/i18n/uk-UA/common.ts b/web/i18n/uk-UA/common.ts index 38929f13d56cb5..bfdaf7c7ca7a94 100644 --- a/web/i18n/uk-UA/common.ts +++ b/web/i18n/uk-UA/common.ts @@ -49,6 +49,7 @@ const translation = { saveAndRegenerate: 'Збереження та регенерація дочірніх фрагментів', submit: 'Представити', skip: 'Корабель', + imageCopied: 'Скопійоване зображення', }, placeholder: { input: 'Будь ласка, введіть текст', diff --git a/web/i18n/uk-UA/dataset.ts b/web/i18n/uk-UA/dataset.ts index e4ec8851ae5fd1..20948e7b997653 100644 --- a/web/i18n/uk-UA/dataset.ts +++ b/web/i18n/uk-UA/dataset.ts @@ -167,6 +167,8 @@ const translation = { documentsDisabled: 'Документи {{num}} вимкнені - неактивні понад 30 днів', localDocs: 'Локальні документи', enable: 'Вмикати', + allKnowledge: 'Всі знання', + allKnowledgeDescription: 'Виберіть, щоб відобразити всі знання в цій робочій області. Тільки власник робочої області може керувати всіма знаннями.', } export default translation diff --git a/web/i18n/vi-VN/common.ts b/web/i18n/vi-VN/common.ts index 7115a86eaed5ab..9fd1af1d44c152 100644 --- a/web/i18n/vi-VN/common.ts +++ b/web/i18n/vi-VN/common.ts @@ -49,6 +49,7 @@ const translation = { viewMore: 'XEM THÊM', submit: 'Trình', skip: 'Tàu', + imageCopied: 'Hình ảnh sao chép', }, placeholder: { input: 'Vui lòng nhập', diff --git a/web/i18n/vi-VN/dataset.ts b/web/i18n/vi-VN/dataset.ts index 0e9ab77d0f743c..1ab84bb85f7ed5 100644 --- a/web/i18n/vi-VN/dataset.ts +++ b/web/i18n/vi-VN/dataset.ts @@ -166,6 +166,8 @@ const translation = { enable: 'Kích hoạt', preprocessDocument: '{{số}} Tiền xử lý tài liệu', documentsDisabled: '{{num}} tài liệu bị vô hiệu hóa - không hoạt động trong hơn 30 ngày', + allKnowledge: 'Tất cả kiến thức', + allKnowledgeDescription: 'Chọn để hiển thị tất cả kiến thức trong không gian làm việc này. Chỉ Chủ sở hữu không gian làm việc mới có thể quản lý tất cả kiến thức.', } export default translation diff --git a/web/i18n/zh-Hant/common.ts b/web/i18n/zh-Hant/common.ts index 32d5e7318df79f..8317fca4ea3080 100644 --- a/web/i18n/zh-Hant/common.ts +++ b/web/i18n/zh-Hant/common.ts @@ -49,6 +49,7 @@ const translation = { regenerate: '再生', submit: '提交', skip: '船', + imageCopied: '複製的圖片', }, placeholder: { input: '請輸入', diff --git a/web/i18n/zh-Hant/dataset.ts b/web/i18n/zh-Hant/dataset.ts index 1aaeb50cd2f8ec..6a0d9cab21ea93 100644 --- a/web/i18n/zh-Hant/dataset.ts +++ b/web/i18n/zh-Hant/dataset.ts @@ -166,6 +166,8 @@ const translation = { documentsDisabled: '已禁用 {{num}} 個文檔 - 處於非活動狀態超過 30 天', localDocs: '本地文件', preprocessDocument: '{{num}}預處理文件', + allKnowledge: '所有知識', + allKnowledgeDescription: '選擇以顯示此工作區中的所有知識。只有 Workspace 擁有者可以管理所有知識。', } export default translation From 989fb11fd715a96576333207189a12d8594c60f1 Mon Sep 17 00:00:00 2001 From: gakkiyomi Date: Thu, 9 Jan 2025 21:30:17 +0800 Subject: [PATCH 28/42] improve the readability of the function generate_api_key (#12552) --- api/models/model.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/models/model.py b/api/models/model.py index d2d4d5853fd2b9..d6f73c5ede8bf7 100644 --- a/api/models/model.py +++ b/api/models/model.py @@ -1405,9 +1405,8 @@ class ApiToken(db.Model): # type: ignore[name-defined] def generate_api_key(prefix, n): while True: result = prefix + generate_string(n) - while db.session.query(ApiToken).filter(ApiToken.token == result).count() > 0: - result = prefix + generate_string(n) - + if db.session.query(ApiToken).filter(ApiToken.token == result).count() > 0: + continue return result From d8f57bf8992148b0dad65a9c304308cc62f6d65c Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Sun, 12 Jan 2025 14:50:46 +0800 Subject: [PATCH 29/42] Feat/new saas billing (#12591) --- api/controllers/console/datasets/datasets.py | 10 +++++- .../console/datasets/datasets_document.py | 10 ++++++ .../console/datasets/datasets_segments.py | 11 +++++++ .../console/datasets/hit_testing.py | 7 +++- api/controllers/console/wraps.py | 33 ++++++++++++++++++- api/controllers/service_api/wraps.py | 31 +++++++++++++++++ .../knowledge_retrieval_node.py | 20 +++++++++++ api/services/billing_service.py | 8 +++++ api/services/feature_service.py | 17 ++++++++++ 9 files changed, 144 insertions(+), 3 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 386e45c58e5042..45c38dba3eadcf 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -10,7 +10,12 @@ from controllers.console.apikey import api_key_fields, api_key_list from controllers.console.app.error import ProviderNotInitializeError from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError -from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required +from controllers.console.wraps import ( + account_initialization_required, + cloud_edition_billing_rate_limit_check, + enterprise_license_required, + setup_required, +) from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError from core.indexing_runner import IndexingRunner from core.model_runtime.entities.model_entities import ModelType @@ -93,6 +98,7 @@ def get(self): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def post(self): parser = reqparse.RequestParser() parser.add_argument( @@ -207,6 +213,7 @@ def get(self, dataset_id): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id): dataset_id_str = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id_str) @@ -310,6 +317,7 @@ def patch(self, dataset_id): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id): dataset_id_str = str(dataset_id) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index c11beaeee1035c..c625b640a7a250 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -27,6 +27,7 @@ ) from controllers.console.wraps import ( account_initialization_required, + cloud_edition_billing_rate_limit_check, cloud_edition_billing_resource_check, setup_required, ) @@ -230,6 +231,7 @@ def get(self, dataset_id): @account_initialization_required @marshal_with(documents_and_batch_fields) @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): dataset_id = str(dataset_id) @@ -285,6 +287,7 @@ def post(self, dataset_id): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -308,6 +311,7 @@ class DatasetInitApi(Resource): @account_initialization_required @marshal_with(dataset_and_document_fields) @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self): # The role of the current user in the ta table must be admin, owner, or editor if not current_user.is_editor: @@ -680,6 +684,7 @@ class DocumentProcessingApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, action): dataset_id = str(dataset_id) document_id = str(document_id) @@ -716,6 +721,7 @@ class DocumentDeleteApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id): dataset_id = str(dataset_id) document_id = str(document_id) @@ -784,6 +790,7 @@ class DocumentStatusApi(DocumentResource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, action): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -879,6 +886,7 @@ class DocumentPauseApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id): """pause document.""" dataset_id = str(dataset_id) @@ -911,6 +919,7 @@ class DocumentRecoverApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id): """recover document.""" dataset_id = str(dataset_id) @@ -940,6 +949,7 @@ class DocumentRetryApi(DocumentResource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): """retry document.""" diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 96654c09fd0223..034fe9cfe208f3 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -19,6 +19,7 @@ from controllers.console.wraps import ( account_initialization_required, cloud_edition_billing_knowledge_limit_check, + cloud_edition_billing_rate_limit_check, cloud_edition_billing_resource_check, setup_required, ) @@ -106,6 +107,7 @@ def get(self, dataset_id, document_id): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -137,6 +139,7 @@ class DatasetDocumentSegmentApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, action): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -192,6 +195,7 @@ class DatasetDocumentSegmentAddApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -242,6 +246,7 @@ class DatasetDocumentSegmentUpdateApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -302,6 +307,7 @@ def patch(self, dataset_id, document_id, segment_id): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -339,6 +345,7 @@ class DatasetDocumentSegmentBatchImportApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -405,6 +412,7 @@ class ChildChunkAddApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -503,6 +511,7 @@ def get(self, dataset_id, document_id, segment_id): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -546,6 +555,7 @@ class ChildChunkUpdateApi(Resource): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id, segment_id, child_chunk_id): # check dataset dataset_id = str(dataset_id) @@ -590,6 +600,7 @@ def delete(self, dataset_id, document_id, segment_id, child_chunk_id): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") + @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id, child_chunk_id): # check dataset dataset_id = str(dataset_id) diff --git a/api/controllers/console/datasets/hit_testing.py b/api/controllers/console/datasets/hit_testing.py index 18b746f547287c..d344e9d1267bfa 100644 --- a/api/controllers/console/datasets/hit_testing.py +++ b/api/controllers/console/datasets/hit_testing.py @@ -2,7 +2,11 @@ from controllers.console import api from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase -from controllers.console.wraps import account_initialization_required, setup_required +from controllers.console.wraps import ( + account_initialization_required, + cloud_edition_billing_rate_limit_check, + setup_required, +) from libs.login import login_required @@ -10,6 +14,7 @@ class HitTestingApi(Resource, DatasetsHitTestingBase): @setup_required @login_required @account_initialization_required + @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): dataset_id_str = str(dataset_id) diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index 111db7ccf2da04..e92c0ae95200ea 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -1,5 +1,6 @@ import json import os +import time from functools import wraps from flask import abort, request @@ -7,6 +8,7 @@ from configs import dify_config from controllers.console.workspace.error import AccountNotInitializedError +from extensions.ext_redis import redis_client from models.model import DifySetup from services.feature_service import FeatureService, LicenseStatus from services.operation_service import OperationService @@ -66,7 +68,9 @@ def decorated(*args, **kwargs): elif resource == "apps" and 0 < apps.limit <= apps.size: abort(403, "The number of apps has reached the limit of your subscription.") elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size: - abort(403, "The capacity of the vector space has reached the limit of your subscription.") + abort( + 403, "The capacity of the knowledge storage space has reached the limit of your subscription." + ) elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size: # The api of file upload is used in the multiple places, # so we need to check the source of the request from datasets @@ -111,6 +115,33 @@ def decorated(*args, **kwargs): return interceptor +def cloud_edition_billing_rate_limit_check(resource: str): + def interceptor(view): + @wraps(view) + def decorated(*args, **kwargs): + if resource == "knowledge": + knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(current_user.current_tenant_id) + if knowledge_rate_limit.enabled: + current_time = int(time.time() * 1000) + key = f"rate_limit_{current_user.current_tenant_id}" + + redis_client.zadd(key, {current_time: current_time}) + + redis_client.zremrangebyscore(key, 0, current_time - 60000) + + request_count = redis_client.zcard(key) + + if request_count > knowledge_rate_limit.limit: + abort( + 403, "Sorry, you have reached the knowledge base request rate limit of your subscription." + ) + return view(*args, **kwargs) + + return decorated + + return interceptor + + def cloud_utm_record(view): @wraps(view) def decorated(*args, **kwargs): diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index 976db1eb46f325..8314a8240c350e 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -1,3 +1,4 @@ +import time from collections.abc import Callable from datetime import UTC, datetime, timedelta from enum import Enum @@ -13,6 +14,7 @@ from werkzeug.exceptions import Forbidden, Unauthorized from extensions.ext_database import db +from extensions.ext_redis import redis_client from libs.login import _get_user from models.account import Account, Tenant, TenantAccountJoin, TenantStatus from models.model import ApiToken, App, EndUser @@ -139,6 +141,35 @@ def decorated(*args, **kwargs): return interceptor +def cloud_edition_billing_rate_limit_check(resource: str, api_token_type: str): + def interceptor(view): + @wraps(view) + def decorated(*args, **kwargs): + api_token = validate_and_get_api_token(api_token_type) + + if resource == "knowledge": + knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(api_token.tenant_id) + if knowledge_rate_limit.enabled: + current_time = int(time.time() * 1000) + key = f"rate_limit_{api_token.tenant_id}" + + redis_client.zadd(key, {current_time: current_time}) + + redis_client.zremrangebyscore(key, 0, current_time - 60000) + + request_count = redis_client.zcard(key) + + if request_count > knowledge_rate_limit.limit: + raise Forbidden( + "Sorry, you have reached the knowledge base request rate limit of your subscription." + ) + return view(*args, **kwargs) + + return decorated + + return interceptor + + def validate_dataset_token(view=None): def decorator(view): @wraps(view) diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index 0f239af51ae79c..be82ad2a8290ab 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -1,4 +1,5 @@ import logging +import time from collections.abc import Mapping, Sequence from typing import Any, cast @@ -19,8 +20,10 @@ from core.workflow.nodes.base import BaseNode from core.workflow.nodes.enums import NodeType from extensions.ext_database import db +from extensions.ext_redis import redis_client from models.dataset import Dataset, Document from models.workflow import WorkflowNodeExecutionStatus +from services.feature_service import FeatureService from .entities import KnowledgeRetrievalNodeData from .exc import ( @@ -61,6 +64,23 @@ def _run(self) -> NodeRunResult: return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, inputs=variables, error="Query is required." ) + # check rate limit + if self.tenant_id: + knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(self.tenant_id) + if knowledge_rate_limit.enabled: + current_time = int(time.time() * 1000) + key = f"rate_limit_{self.tenant_id}" + redis_client.zadd(key, {current_time: current_time}) + redis_client.zremrangebyscore(key, 0, current_time - 60000) + request_count = redis_client.zcard(key) + if request_count > knowledge_rate_limit.limit: + return NodeRunResult( + status=WorkflowNodeExecutionStatus.FAILED, + inputs=variables, + error="Sorry, you have reached the knowledge base request rate limit of your subscription.", + error_type="RateLimitExceeded", + ) + # retrieve knowledge try: results = self._fetch_dataset_retriever(node_data=self.node_data, query=query) diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 0d50a2aa8c34ff..9d9dd8a368e7c3 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -19,6 +19,14 @@ def get_info(cls, tenant_id: str): billing_info = cls._send_request("GET", "/subscription/info", params=params) return billing_info + @classmethod + def get_knowledge_rate_limit(cls, tenant_id: str): + params = {"tenant_id": tenant_id} + + knowledge_rate_limit = cls._send_request("GET", "/subscription/knowledge-rate-limit", params=params) + + return knowledge_rate_limit.get("limit", 10) + @classmethod def get_subscription(cls, plan: str, interval: str, prefilled_email: str = "", tenant_id: str = ""): params = {"plan": plan, "interval": interval, "prefilled_email": prefilled_email, "tenant_id": tenant_id} diff --git a/api/services/feature_service.py b/api/services/feature_service.py index b9261d19d7930e..52cfe4f2cb7e53 100644 --- a/api/services/feature_service.py +++ b/api/services/feature_service.py @@ -41,6 +41,7 @@ class FeatureModel(BaseModel): members: LimitationModel = LimitationModel(size=0, limit=1) apps: LimitationModel = LimitationModel(size=0, limit=10) vector_space: LimitationModel = LimitationModel(size=0, limit=5) + knowledge_rate_limit: int = 10 annotation_quota_limit: LimitationModel = LimitationModel(size=0, limit=10) documents_upload_quota: LimitationModel = LimitationModel(size=0, limit=50) docs_processing: str = "standard" @@ -52,6 +53,11 @@ class FeatureModel(BaseModel): model_config = ConfigDict(protected_namespaces=()) +class KnowledgeRateLimitModel(BaseModel): + enabled: bool = False + limit: int = 10 + + class SystemFeatureModel(BaseModel): sso_enforced_for_signin: bool = False sso_enforced_for_signin_protocol: str = "" @@ -79,6 +85,14 @@ def get_features(cls, tenant_id: str) -> FeatureModel: return features + @classmethod + def get_knowledge_rate_limit(cls, tenant_id: str): + knowledge_rate_limit = KnowledgeRateLimitModel() + if dify_config.BILLING_ENABLED and tenant_id: + knowledge_rate_limit.enabled = True + knowledge_rate_limit.limit = BillingService.get_knowledge_rate_limit(tenant_id) + return knowledge_rate_limit + @classmethod def get_system_features(cls) -> SystemFeatureModel: system_features = SystemFeatureModel() @@ -144,6 +158,9 @@ def _fulfill_params_from_billing_api(cls, features: FeatureModel, tenant_id: str if "model_load_balancing_enabled" in billing_info: features.model_load_balancing_enabled = billing_info["model_load_balancing_enabled"] + if "knowledge_rate_limit" in billing_info: + features.knowledge_rate_limit = billing_info["knowledge_rate_limit"]["limit"] + @classmethod def _fulfill_params_from_enterprise(cls, features): enterprise_info = EnterpriseService.get_info() From cd257b91c54231d1a4f32c1ad7ff8a0db67b586b Mon Sep 17 00:00:00 2001 From: CN-P5 Date: Mon, 13 Jan 2025 09:06:59 +0800 Subject: [PATCH 30/42] Fix pandas indexing method for knowledge base imports (#12637) (#12638) Co-authored-by: CN-P5 --- api/controllers/console/datasets/datasets_segments.py | 4 ++-- api/core/rag/index_processor/processor/qa_index_processor.py | 2 +- api/services/annotation_service.py | 2 +- api/tasks/batch_create_segment_to_index_task.py | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 034fe9cfe208f3..2dd86a1b32d996 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -375,9 +375,9 @@ def post(self, dataset_id, document_id): result = [] for index, row in df.iterrows(): if document.doc_form == "qa_model": - data = {"content": row[0], "answer": row[1]} + data = {"content": row.iloc[0], "answer": row.iloc[1]} else: - data = {"content": row[0]} + data = {"content": row.iloc[0]} result.append(data) if len(result) == 0: raise ValueError("The CSV file is empty.") diff --git a/api/core/rag/index_processor/processor/qa_index_processor.py b/api/core/rag/index_processor/processor/qa_index_processor.py index 58b50a9fcbc67e..0055625e136c79 100644 --- a/api/core/rag/index_processor/processor/qa_index_processor.py +++ b/api/core/rag/index_processor/processor/qa_index_processor.py @@ -112,7 +112,7 @@ def format_by_template(self, file: FileStorage, **kwargs) -> list[Document]: df = pd.read_csv(file) text_docs = [] for index, row in df.iterrows(): - data = Document(page_content=row[0], metadata={"answer": row[1]}) + data = Document(page_content=row.iloc[0], metadata={"answer": row.iloc[1]}) text_docs.append(data) if len(text_docs) == 0: raise ValueError("The CSV file is empty.") diff --git a/api/services/annotation_service.py b/api/services/annotation_service.py index a946405c955cec..45ec1e9b5aec61 100644 --- a/api/services/annotation_service.py +++ b/api/services/annotation_service.py @@ -286,7 +286,7 @@ def batch_import_app_annotations(cls, app_id, file: FileStorage) -> dict: df = pd.read_csv(file) result = [] for index, row in df.iterrows(): - content = {"question": row[0], "answer": row[1]} + content = {"question": row.iloc[0], "answer": row.iloc[1]} result.append(content) if len(result) == 0: raise ValueError("The CSV file is empty.") diff --git a/api/tasks/batch_create_segment_to_index_task.py b/api/tasks/batch_create_segment_to_index_task.py index 05a0f0a407f5f3..dbef6b708e4f29 100644 --- a/api/tasks/batch_create_segment_to_index_task.py +++ b/api/tasks/batch_create_segment_to_index_task.py @@ -77,8 +77,8 @@ def batch_create_segment_to_index_task( index_node_id=doc_id, index_node_hash=segment_hash, position=max_position + 1 if max_position else 1, - content=content, - word_count=len(content), + content=content_str, + word_count=len(content_str), tokens=tokens, created_by=user_id, indexing_at=datetime.datetime.now(datetime.UTC).replace(tzinfo=None), From a6455269f001cf584de8c1b2104eb3f97d4b739e Mon Sep 17 00:00:00 2001 From: Chuehnone <1897025+chuehnone@users.noreply.github.com> Date: Mon, 13 Jan 2025 09:12:43 +0800 Subject: [PATCH 31/42] chore: Adjust translations to align with Taiwanese Mandarin conventions (#12633) --- web/i18n/zh-Hant/app.ts | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/web/i18n/zh-Hant/app.ts b/web/i18n/zh-Hant/app.ts index 99de5042c24fdf..0cf916f3ff811c 100644 --- a/web/i18n/zh-Hant/app.ts +++ b/web/i18n/zh-Hant/app.ts @@ -17,7 +17,7 @@ const translation = { createFromConfigFile: '透過 DSL 檔案建立', deleteAppConfirmTitle: '確認刪除應用?', deleteAppConfirmContent: - '刪除應用將無法撤銷。使用者將不能訪問你的應用，所有 Prompt 編排配置和日誌均將一併被刪除。', + '刪除應用將無法復原。使用者將無法存取你的應用，所有 Prompt 設定和日誌都將一併被刪除。', appDeleted: '應用已刪除', appDeleteFailed: '應用刪除失敗', join: '參與社群', @@ -26,12 +26,12 @@ const translation = { newApp: { startFromBlank: '建立空白應用', startFromTemplate: '從應用模版建立', - captionAppType: '想要哪種應用型別？', - chatbotDescription: '使用大型語言模型構建基於聊天的助手', - completionDescription: '構建一個根據提示生成高質量文字的應用程式，例如生成文章、摘要、翻譯等。', - completionWarning: '該型別不久後將不再支援建立', + captionAppType: '想要哪種應用類型？', + chatbotDescription: '使用大型語言模型構建聊天助手', + completionDescription: '構建一個根據提示生成高品質文字的應用程式，例如生成文章、摘要、翻譯等。', + completionWarning: '該類型不久後將不再支援建立', agentDescription: '構建一個智慧Agent，可以自主選擇工具來完成任務', - workflowDescription: '以工作流的形式編排生成型應用，提供更多的自定義能力。 它適合有經驗的使用者。', + workflowDescription: '以工作流的形式編排生成型應用，提供更多的自訂設定。 它適合有經驗的使用者。', workflowWarning: '正在進行 Beta 測試', chatbotType: '聊天助手編排方法', basic: '基礎編排', @@ -40,7 +40,7 @@ const translation = { basicDescription: '基本編排允許使用簡單的設定編排聊天機器人應用程式，而無需修改內建提示。 它適合初學者。', advanced: '工作流編排', advancedFor: '進階使用者適用', - advancedDescription: '工作流編排以工作流的形式編排聊天機器人，提供高度的自定義，包括編輯內建提示的能力。 它適合有經驗的使用者。', + advancedDescription: '工作流編排以工作流的形式編排聊天機器人，提供自訂設定，包括編輯內建提示的能力。 它適合有經驗的使用者。', captionName: '應用名稱 & 圖示', appNamePlaceholder: '給你的應用起個名字', captionDescription: '描述', @@ -53,14 +53,14 @@ const translation = { agentAssistant: '新的智慧助手', completeApp: '文字生成應用', completeAppIntro: - '我要構建一個根據提示生成高質量文字的應用，例如生成文章、摘要、翻譯等', + '我要構建一個根據提示生成高品質文字的應用，例如生成文章、摘要、翻譯等', showTemplates: '我想從範例模板中選擇', - hideTemplates: '返回應用型別選擇', + hideTemplates: '返回應用類型選擇', Create: '建立', Cancel: '取消', nameNotEmpty: '名稱不能為空', appTemplateNotSelected: '請選擇應用模版', - appTypeRequired: '請選擇應用型別', + appTypeRequired: '請選擇應用類型', appCreated: '應用已建立', appCreateFailed: '應用建立失敗', caution: '謹慎', @@ -111,7 +111,7 @@ const translation = { removeOriginal: '刪除原應用', switchStart: '開始遷移', typeSelector: { - all: '所有型別', + all: '所有類型', chatbot: '聊天助手', agent: 'Agent', workflow: '工作流', @@ -150,7 +150,7 @@ const translation = { project: '專案', publicKey: '公鑰', secretKey: '密鑰', - viewDocsLink: '查看{{key}}文檔', + viewDocsLink: '查看{{key}}文件', removeConfirmTitle: '移除{{key}}配置？', removeConfirmContent: '當前配置正在使用中，移除它將關閉追蹤功能。', }, @@ -163,7 +163,7 @@ const translation = { importFromDSLUrl: '寄件者 URL', importFromDSL: '從 DSL 導入', importFromDSLFile: '從 DSL 檔', - importFromDSLUrlPlaceholder: '在此處粘貼 DSL 連結', + importFromDSLUrlPlaceholder: '在此處貼上 DSL 連結', mermaid: { handDrawn: '手繪', classic: '經典', @@ -182,7 +182,7 @@ const translation = { searchAllTemplate: '搜尋所有樣本...', byCategories: '按類別', }, - showMyCreatedAppsOnly: '我创建的', + showMyCreatedAppsOnly: '我建立的', } export default translation From 4e101604c350bce9305c3e4c8a6e9c0d5c77e230 Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 13 Jan 2025 09:38:48 +0800 Subject: [PATCH 32/42] fix: ruff check for True if ... else (#12576) Signed-off-by: yihong0618 --- api/.ruff.toml | 1 - api/controllers/console/explore/conversation.py | 2 +- api/controllers/service_api/wraps.py | 2 +- api/controllers/web/conversation.py | 2 +- api/core/model_runtime/schema_validators/common_validator.py | 2 +- api/core/rag/datasource/vdb/lindorm/lindorm_vector.py | 4 ++-- 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/api/.ruff.toml b/api/.ruff.toml index 89a2da35d6cf87..e319ed4d5417f7 100644 --- a/api/.ruff.toml +++ b/api/.ruff.toml @@ -69,7 +69,6 @@ ignore = [ "SIM108", # if-else-block-instead-of-if-exp "SIM113", # enumerate-for-loop "SIM117", # multiple-with-statements - "SIM210", # if-expr-with-true-false ] [lint.per-file-ignores] diff --git a/api/controllers/console/explore/conversation.py b/api/controllers/console/explore/conversation.py index 91916cbc1ed85f..600e78e09e3c2c 100644 --- a/api/controllers/console/explore/conversation.py +++ b/api/controllers/console/explore/conversation.py @@ -32,7 +32,7 @@ def get(self, installed_app): pinned = None if "pinned" in args and args["pinned"] is not None: - pinned = True if args["pinned"] == "true" else False + pinned = args["pinned"] == "true" try: with Session(db.engine) as session: diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index 8314a8240c350e..43f718306bfef2 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -267,7 +267,7 @@ def create_or_update_end_user_for_user_id(app_model: App, user_id: Optional[str] tenant_id=app_model.tenant_id, app_id=app_model.id, type="service_api", - is_anonymous=True if user_id == "DEFAULT-USER" else False, + is_anonymous=user_id == "DEFAULT-USER", session_id=user_id, ) db.session.add(end_user) diff --git a/api/controllers/web/conversation.py b/api/controllers/web/conversation.py index 28feb1ca47effd..419247ea14e51b 100644 --- a/api/controllers/web/conversation.py +++ b/api/controllers/web/conversation.py @@ -39,7 +39,7 @@ def get(self, app_model, end_user): pinned = None if "pinned" in args and args["pinned"] is not None: - pinned = True if args["pinned"] == "true" else False + pinned = args["pinned"] == "true" try: with Session(db.engine) as session: diff --git a/api/core/model_runtime/schema_validators/common_validator.py b/api/core/model_runtime/schema_validators/common_validator.py index 8cc8adfc3656ea..810a7c4c44a34b 100644 --- a/api/core/model_runtime/schema_validators/common_validator.py +++ b/api/core/model_runtime/schema_validators/common_validator.py @@ -87,6 +87,6 @@ def _validate_credential_form_schema( if value.lower() not in {"true", "false"}: raise ValueError(f"Variable {credential_form_schema.variable} should be true or false") - value = True if value.lower() == "true" else False + value = value.lower() == "true" return value diff --git a/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py b/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py index d7a14207e9375a..66fba763e731c1 100644 --- a/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py +++ b/api/core/rag/datasource/vdb/lindorm/lindorm_vector.py @@ -258,7 +258,7 @@ def create_collection(self, dimension: int, **kwargs): hnsw_ef_construction = kwargs.pop("hnsw_ef_construction", 500) ivfpq_m = kwargs.pop("ivfpq_m", dimension) nlist = kwargs.pop("nlist", 1000) - centroids_use_hnsw = kwargs.pop("centroids_use_hnsw", True if nlist >= 5000 else False) + centroids_use_hnsw = kwargs.pop("centroids_use_hnsw", nlist >= 5000) centroids_hnsw_m = kwargs.pop("centroids_hnsw_m", 24) centroids_hnsw_ef_construct = kwargs.pop("centroids_hnsw_ef_construct", 500) centroids_hnsw_ef_search = kwargs.pop("centroids_hnsw_ef_search", 100) @@ -305,7 +305,7 @@ def default_text_mapping(dimension: int, method_name: str, **kwargs: Any) -> dic if method_name == "ivfpq": ivfpq_m = kwargs["ivfpq_m"] nlist = kwargs["nlist"] - centroids_use_hnsw = True if nlist > 10000 else False + centroids_use_hnsw = nlist > 10000 centroids_hnsw_m = 24 centroids_hnsw_ef_construct = 500 centroids_hnsw_ef_search = 100 From 831459b895c62437702b8f46594e0f18ab41f00f Mon Sep 17 00:00:00 2001 From: yihong Date: Mon, 13 Jan 2025 09:55:55 +0800 Subject: [PATCH 33/42] fix: ruff with statements (#12578) Signed-off-by: yihong0618 Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com> --- api/.ruff.toml | 1 + .../integration_tests/controllers/test_controllers.py | 7 +++---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/.ruff.toml b/api/.ruff.toml index e319ed4d5417f7..89a2da35d6cf87 100644 --- a/api/.ruff.toml +++ b/api/.ruff.toml @@ -69,6 +69,7 @@ ignore = [ "SIM108", # if-else-block-instead-of-if-exp "SIM113", # enumerate-for-loop "SIM117", # multiple-with-statements + "SIM210", # if-expr-with-true-false ] [lint.per-file-ignores] diff --git a/api/tests/integration_tests/controllers/test_controllers.py b/api/tests/integration_tests/controllers/test_controllers.py index 5e3ee6bedc7ebb..276ad3a7ed818a 100644 --- a/api/tests/integration_tests/controllers/test_controllers.py +++ b/api/tests/integration_tests/controllers/test_controllers.py @@ -4,7 +4,6 @@ def test_post_requires_login(app): - with app.test_client() as client: - with patch("flask_login.utils._get_user", mock_user): - response = client.get("/console/api/data-source/integrates") - assert response.status_code == 200 + with app.test_client() as client, patch("flask_login.utils._get_user", mock_user): + response = client.get("/console/api/data-source/integrates") + assert response.status_code == 200 From 54b5b80a07c117237c199279f19b0ed7e4bf4ce9 Mon Sep 17 00:00:00 2001 From: Kevin9703 <51311316+Kevin9703@users.noreply.github.com> Date: Mon, 13 Jan 2025 14:54:21 +0800 Subject: [PATCH 34/42] fix(workflow): fix answer node stream processing in conditional branches (#12510) --- .../nodes/answer/base_stream_processor.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/api/core/workflow/nodes/answer/base_stream_processor.py b/api/core/workflow/nodes/answer/base_stream_processor.py index f22ea078fb87eb..4759356ae124a4 100644 --- a/api/core/workflow/nodes/answer/base_stream_processor.py +++ b/api/core/workflow/nodes/answer/base_stream_processor.py @@ -1,6 +1,7 @@ import logging from abc import ABC, abstractmethod from collections.abc import Generator +from typing import Optional from core.workflow.entities.variable_pool import VariablePool from core.workflow.graph_engine.entities.event import GraphEngineEvent, NodeRunExceptionEvent, NodeRunSucceededEvent @@ -48,25 +49,35 @@ def _remove_unreachable_nodes(self, event: NodeRunSucceededEvent | NodeRunExcept # we remove the node maybe shortcut the answer node, so comment this code for now # there is not effect on the answer node and the workflow, when we have a better solution # we can open this code. Issues: #11542 #9560 #10638 #10564 - ids = self._fetch_node_ids_in_reachable_branch(edge.target_node_id) - if "answer" in ids: - continue - else: - reachable_node_ids.extend(ids) + # ids = self._fetch_node_ids_in_reachable_branch(edge.target_node_id) + # if "answer" in ids: + # continue + # else: + # reachable_node_ids.extend(ids) + + # The branch_identify parameter is added to ensure that + # only nodes in the correct logical branch are included. + ids = self._fetch_node_ids_in_reachable_branch(edge.target_node_id, run_result.edge_source_handle) + reachable_node_ids.extend(ids) else: unreachable_first_node_ids.append(edge.target_node_id) for node_id in unreachable_first_node_ids: self._remove_node_ids_in_unreachable_branch(node_id, reachable_node_ids) - def _fetch_node_ids_in_reachable_branch(self, node_id: str) -> list[str]: + def _fetch_node_ids_in_reachable_branch(self, node_id: str, branch_identify: Optional[str] = None) -> list[str]: node_ids = [] for edge in self.graph.edge_mapping.get(node_id, []): if edge.target_node_id == self.graph.root_node_id: continue + # Only follow edges that match the branch_identify or have no run_condition + if edge.run_condition and edge.run_condition.branch_identify: + if not branch_identify or edge.run_condition.branch_identify != branch_identify: + continue + node_ids.append(edge.target_node_id) - node_ids.extend(self._fetch_node_ids_in_reachable_branch(edge.target_node_id)) + node_ids.extend(self._fetch_node_ids_in_reachable_branch(edge.target_node_id, branch_identify)) return node_ids def _remove_node_ids_in_unreachable_branch(self, node_id: str, reachable_node_ids: list[str]) -> None: From 9a6b1dc3a1ea50e7b053f6b5f5a8d0021bedddcb Mon Sep 17 00:00:00 2001 From: Jyong <76649700+JohnJyong@users.noreply.github.com> Date: Mon, 13 Jan 2025 15:17:43 +0800 Subject: [PATCH 35/42] Revert "Feat/new saas billing" (#12673) --- api/controllers/console/datasets/datasets.py | 10 +----- .../console/datasets/datasets_document.py | 10 ------ .../console/datasets/datasets_segments.py | 11 ------- .../console/datasets/hit_testing.py | 7 +--- api/controllers/console/wraps.py | 33 +------------------ api/controllers/service_api/wraps.py | 31 ----------------- .../knowledge_retrieval_node.py | 20 ----------- api/services/billing_service.py | 8 ----- api/services/feature_service.py | 17 ---------- 9 files changed, 3 insertions(+), 144 deletions(-) diff --git a/api/controllers/console/datasets/datasets.py b/api/controllers/console/datasets/datasets.py index 45c38dba3eadcf..386e45c58e5042 100644 --- a/api/controllers/console/datasets/datasets.py +++ b/api/controllers/console/datasets/datasets.py @@ -10,12 +10,7 @@ from controllers.console.apikey import api_key_fields, api_key_list from controllers.console.app.error import ProviderNotInitializeError from controllers.console.datasets.error import DatasetInUseError, DatasetNameDuplicateError, IndexingEstimateError -from controllers.console.wraps import ( - account_initialization_required, - cloud_edition_billing_rate_limit_check, - enterprise_license_required, - setup_required, -) +from controllers.console.wraps import account_initialization_required, enterprise_license_required, setup_required from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError from core.indexing_runner import IndexingRunner from core.model_runtime.entities.model_entities import ModelType @@ -98,7 +93,6 @@ def get(self): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def post(self): parser = reqparse.RequestParser() parser.add_argument( @@ -213,7 +207,6 @@ def get(self, dataset_id): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id): dataset_id_str = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id_str) @@ -317,7 +310,6 @@ def patch(self, dataset_id): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id): dataset_id_str = str(dataset_id) diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py index c625b640a7a250..c11beaeee1035c 100644 --- a/api/controllers/console/datasets/datasets_document.py +++ b/api/controllers/console/datasets/datasets_document.py @@ -27,7 +27,6 @@ ) from controllers.console.wraps import ( account_initialization_required, - cloud_edition_billing_rate_limit_check, cloud_edition_billing_resource_check, setup_required, ) @@ -231,7 +230,6 @@ def get(self, dataset_id): @account_initialization_required @marshal_with(documents_and_batch_fields) @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): dataset_id = str(dataset_id) @@ -287,7 +285,6 @@ def post(self, dataset_id): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -311,7 +308,6 @@ class DatasetInitApi(Resource): @account_initialization_required @marshal_with(dataset_and_document_fields) @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self): # The role of the current user in the ta table must be admin, owner, or editor if not current_user.is_editor: @@ -684,7 +680,6 @@ class DocumentProcessingApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, action): dataset_id = str(dataset_id) document_id = str(document_id) @@ -721,7 +716,6 @@ class DocumentDeleteApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id): dataset_id = str(dataset_id) document_id = str(document_id) @@ -790,7 +784,6 @@ class DocumentStatusApi(DocumentResource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, action): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -886,7 +879,6 @@ class DocumentPauseApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id): """pause document.""" dataset_id = str(dataset_id) @@ -919,7 +911,6 @@ class DocumentRecoverApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id): """recover document.""" dataset_id = str(dataset_id) @@ -949,7 +940,6 @@ class DocumentRetryApi(DocumentResource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): """retry document.""" diff --git a/api/controllers/console/datasets/datasets_segments.py b/api/controllers/console/datasets/datasets_segments.py index 2dd86a1b32d996..d48dbe1772c353 100644 --- a/api/controllers/console/datasets/datasets_segments.py +++ b/api/controllers/console/datasets/datasets_segments.py @@ -19,7 +19,6 @@ from controllers.console.wraps import ( account_initialization_required, cloud_edition_billing_knowledge_limit_check, - cloud_edition_billing_rate_limit_check, cloud_edition_billing_resource_check, setup_required, ) @@ -107,7 +106,6 @@ def get(self, dataset_id, document_id): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -139,7 +137,6 @@ class DatasetDocumentSegmentApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, action): dataset_id = str(dataset_id) dataset = DatasetService.get_dataset(dataset_id) @@ -195,7 +192,6 @@ class DatasetDocumentSegmentAddApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -246,7 +242,6 @@ class DatasetDocumentSegmentUpdateApi(Resource): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -307,7 +302,6 @@ def patch(self, dataset_id, document_id, segment_id): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -345,7 +339,6 @@ class DatasetDocumentSegmentBatchImportApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id): # check dataset dataset_id = str(dataset_id) @@ -412,7 +405,6 @@ class ChildChunkAddApi(Resource): @account_initialization_required @cloud_edition_billing_resource_check("vector_space") @cloud_edition_billing_knowledge_limit_check("add_segment") - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -511,7 +503,6 @@ def get(self, dataset_id, document_id, segment_id): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id): # check dataset dataset_id = str(dataset_id) @@ -555,7 +546,6 @@ class ChildChunkUpdateApi(Resource): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def delete(self, dataset_id, document_id, segment_id, child_chunk_id): # check dataset dataset_id = str(dataset_id) @@ -600,7 +590,6 @@ def delete(self, dataset_id, document_id, segment_id, child_chunk_id): @login_required @account_initialization_required @cloud_edition_billing_resource_check("vector_space") - @cloud_edition_billing_rate_limit_check("knowledge") def patch(self, dataset_id, document_id, segment_id, child_chunk_id): # check dataset dataset_id = str(dataset_id) diff --git a/api/controllers/console/datasets/hit_testing.py b/api/controllers/console/datasets/hit_testing.py index d344e9d1267bfa..18b746f547287c 100644 --- a/api/controllers/console/datasets/hit_testing.py +++ b/api/controllers/console/datasets/hit_testing.py @@ -2,11 +2,7 @@ from controllers.console import api from controllers.console.datasets.hit_testing_base import DatasetsHitTestingBase -from controllers.console.wraps import ( - account_initialization_required, - cloud_edition_billing_rate_limit_check, - setup_required, -) +from controllers.console.wraps import account_initialization_required, setup_required from libs.login import login_required @@ -14,7 +10,6 @@ class HitTestingApi(Resource, DatasetsHitTestingBase): @setup_required @login_required @account_initialization_required - @cloud_edition_billing_rate_limit_check("knowledge") def post(self, dataset_id): dataset_id_str = str(dataset_id) diff --git a/api/controllers/console/wraps.py b/api/controllers/console/wraps.py index e92c0ae95200ea..111db7ccf2da04 100644 --- a/api/controllers/console/wraps.py +++ b/api/controllers/console/wraps.py @@ -1,6 +1,5 @@ import json import os -import time from functools import wraps from flask import abort, request @@ -8,7 +7,6 @@ from configs import dify_config from controllers.console.workspace.error import AccountNotInitializedError -from extensions.ext_redis import redis_client from models.model import DifySetup from services.feature_service import FeatureService, LicenseStatus from services.operation_service import OperationService @@ -68,9 +66,7 @@ def decorated(*args, **kwargs): elif resource == "apps" and 0 < apps.limit <= apps.size: abort(403, "The number of apps has reached the limit of your subscription.") elif resource == "vector_space" and 0 < vector_space.limit <= vector_space.size: - abort( - 403, "The capacity of the knowledge storage space has reached the limit of your subscription." - ) + abort(403, "The capacity of the vector space has reached the limit of your subscription.") elif resource == "documents" and 0 < documents_upload_quota.limit <= documents_upload_quota.size: # The api of file upload is used in the multiple places, # so we need to check the source of the request from datasets @@ -115,33 +111,6 @@ def decorated(*args, **kwargs): return interceptor -def cloud_edition_billing_rate_limit_check(resource: str): - def interceptor(view): - @wraps(view) - def decorated(*args, **kwargs): - if resource == "knowledge": - knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(current_user.current_tenant_id) - if knowledge_rate_limit.enabled: - current_time = int(time.time() * 1000) - key = f"rate_limit_{current_user.current_tenant_id}" - - redis_client.zadd(key, {current_time: current_time}) - - redis_client.zremrangebyscore(key, 0, current_time - 60000) - - request_count = redis_client.zcard(key) - - if request_count > knowledge_rate_limit.limit: - abort( - 403, "Sorry, you have reached the knowledge base request rate limit of your subscription." - ) - return view(*args, **kwargs) - - return decorated - - return interceptor - - def cloud_utm_record(view): @wraps(view) def decorated(*args, **kwargs): diff --git a/api/controllers/service_api/wraps.py b/api/controllers/service_api/wraps.py index 43f718306bfef2..fc4cce487691c5 100644 --- a/api/controllers/service_api/wraps.py +++ b/api/controllers/service_api/wraps.py @@ -1,4 +1,3 @@ -import time from collections.abc import Callable from datetime import UTC, datetime, timedelta from enum import Enum @@ -14,7 +13,6 @@ from werkzeug.exceptions import Forbidden, Unauthorized from extensions.ext_database import db -from extensions.ext_redis import redis_client from libs.login import _get_user from models.account import Account, Tenant, TenantAccountJoin, TenantStatus from models.model import ApiToken, App, EndUser @@ -141,35 +139,6 @@ def decorated(*args, **kwargs): return interceptor -def cloud_edition_billing_rate_limit_check(resource: str, api_token_type: str): - def interceptor(view): - @wraps(view) - def decorated(*args, **kwargs): - api_token = validate_and_get_api_token(api_token_type) - - if resource == "knowledge": - knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(api_token.tenant_id) - if knowledge_rate_limit.enabled: - current_time = int(time.time() * 1000) - key = f"rate_limit_{api_token.tenant_id}" - - redis_client.zadd(key, {current_time: current_time}) - - redis_client.zremrangebyscore(key, 0, current_time - 60000) - - request_count = redis_client.zcard(key) - - if request_count > knowledge_rate_limit.limit: - raise Forbidden( - "Sorry, you have reached the knowledge base request rate limit of your subscription." - ) - return view(*args, **kwargs) - - return decorated - - return interceptor - - def validate_dataset_token(view=None): def decorator(view): @wraps(view) diff --git a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py index be82ad2a8290ab..0f239af51ae79c 100644 --- a/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py +++ b/api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py @@ -1,5 +1,4 @@ import logging -import time from collections.abc import Mapping, Sequence from typing import Any, cast @@ -20,10 +19,8 @@ from core.workflow.nodes.base import BaseNode from core.workflow.nodes.enums import NodeType from extensions.ext_database import db -from extensions.ext_redis import redis_client from models.dataset import Dataset, Document from models.workflow import WorkflowNodeExecutionStatus -from services.feature_service import FeatureService from .entities import KnowledgeRetrievalNodeData from .exc import ( @@ -64,23 +61,6 @@ def _run(self) -> NodeRunResult: return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, inputs=variables, error="Query is required." ) - # check rate limit - if self.tenant_id: - knowledge_rate_limit = FeatureService.get_knowledge_rate_limit(self.tenant_id) - if knowledge_rate_limit.enabled: - current_time = int(time.time() * 1000) - key = f"rate_limit_{self.tenant_id}" - redis_client.zadd(key, {current_time: current_time}) - redis_client.zremrangebyscore(key, 0, current_time - 60000) - request_count = redis_client.zcard(key) - if request_count > knowledge_rate_limit.limit: - return NodeRunResult( - status=WorkflowNodeExecutionStatus.FAILED, - inputs=variables, - error="Sorry, you have reached the knowledge base request rate limit of your subscription.", - error_type="RateLimitExceeded", - ) - # retrieve knowledge try: results = self._fetch_dataset_retriever(node_data=self.node_data, query=query) diff --git a/api/services/billing_service.py b/api/services/billing_service.py index 9d9dd8a368e7c3..0d50a2aa8c34ff 100644 --- a/api/services/billing_service.py +++ b/api/services/billing_service.py @@ -19,14 +19,6 @@ def get_info(cls, tenant_id: str): billing_info = cls._send_request("GET", "/subscription/info", params=params) return billing_info - @classmethod - def get_knowledge_rate_limit(cls, tenant_id: str): - params = {"tenant_id": tenant_id} - - knowledge_rate_limit = cls._send_request("GET", "/subscription/knowledge-rate-limit", params=params) - - return knowledge_rate_limit.get("limit", 10) - @classmethod def get_subscription(cls, plan: str, interval: str, prefilled_email: str = "", tenant_id: str = ""): params = {"plan": plan, "interval": interval, "prefilled_email": prefilled_email, "tenant_id": tenant_id} diff --git a/api/services/feature_service.py b/api/services/feature_service.py index 52cfe4f2cb7e53..b9261d19d7930e 100644 --- a/api/services/feature_service.py +++ b/api/services/feature_service.py @@ -41,7 +41,6 @@ class FeatureModel(BaseModel): members: LimitationModel = LimitationModel(size=0, limit=1) apps: LimitationModel = LimitationModel(size=0, limit=10) vector_space: LimitationModel = LimitationModel(size=0, limit=5) - knowledge_rate_limit: int = 10 annotation_quota_limit: LimitationModel = LimitationModel(size=0, limit=10) documents_upload_quota: LimitationModel = LimitationModel(size=0, limit=50) docs_processing: str = "standard" @@ -53,11 +52,6 @@ class FeatureModel(BaseModel): model_config = ConfigDict(protected_namespaces=()) -class KnowledgeRateLimitModel(BaseModel): - enabled: bool = False - limit: int = 10 - - class SystemFeatureModel(BaseModel): sso_enforced_for_signin: bool = False sso_enforced_for_signin_protocol: str = "" @@ -85,14 +79,6 @@ def get_features(cls, tenant_id: str) -> FeatureModel: return features - @classmethod - def get_knowledge_rate_limit(cls, tenant_id: str): - knowledge_rate_limit = KnowledgeRateLimitModel() - if dify_config.BILLING_ENABLED and tenant_id: - knowledge_rate_limit.enabled = True - knowledge_rate_limit.limit = BillingService.get_knowledge_rate_limit(tenant_id) - return knowledge_rate_limit - @classmethod def get_system_features(cls) -> SystemFeatureModel: system_features = SystemFeatureModel() @@ -158,9 +144,6 @@ def _fulfill_params_from_billing_api(cls, features: FeatureModel, tenant_id: str if "model_load_balancing_enabled" in billing_info: features.model_load_balancing_enabled = billing_info["model_load_balancing_enabled"] - if "knowledge_rate_limit" in billing_info: - features.knowledge_rate_limit = billing_info["knowledge_rate_limit"]["limit"] - @classmethod def _fulfill_params_from_enterprise(cls, features): enterprise_info = EnterpriseService.get_info() From c700364e1ccd61da79e9b03c993e9eb38348eea3 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Mon, 13 Jan 2025 15:54:26 +0800 Subject: [PATCH 36/42] fix: Update variable handling in VariableAssignerNode and clean up app_dsl_service (#12672) Signed-off-by: -LAN- --- .../workflow/nodes/variable_assigner/v2/node.py | 16 ++++++++++++---- api/services/app_dsl_service.py | 11 +---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/api/core/workflow/nodes/variable_assigner/v2/node.py b/api/core/workflow/nodes/variable_assigner/v2/node.py index 0c4aae827c0a0f..afa5656f46e692 100644 --- a/api/core/workflow/nodes/variable_assigner/v2/node.py +++ b/api/core/workflow/nodes/variable_assigner/v2/node.py @@ -1,4 +1,5 @@ import json +from collections.abc import Sequence from typing import Any, cast from core.variables import SegmentType, Variable @@ -31,7 +32,7 @@ def _run(self) -> NodeRunResult: inputs = self.node_data.model_dump() process_data: dict[str, Any] = {} # NOTE: This node has no outputs - updated_variables: list[Variable] = [] + updated_variable_selectors: list[Sequence[str]] = [] try: for item in self.node_data.items: @@ -98,7 +99,8 @@ def _run(self) -> NodeRunResult: value=item.value, ) variable = variable.model_copy(update={"value": updated_value}) - updated_variables.append(variable) + self.graph_runtime_state.variable_pool.add(variable.selector, variable) + updated_variable_selectors.append(variable.selector) except VariableOperatorNodeError as e: return NodeRunResult( status=WorkflowNodeExecutionStatus.FAILED, @@ -107,9 +109,15 @@ def _run(self) -> NodeRunResult: error=str(e), ) + # The `updated_variable_selectors` is a list contains list[str] which not hashable, + # remove the duplicated items first. + updated_variable_selectors = list(set(map(tuple, updated_variable_selectors))) + # Update variables - for variable in updated_variables: - self.graph_runtime_state.variable_pool.add(variable.selector, variable) + for selector in updated_variable_selectors: + variable = self.graph_runtime_state.variable_pool.get(selector) + if not isinstance(variable, Variable): + raise VariableNotFoundError(variable_selector=selector) process_data[variable.name] = variable.value if variable.selector[0] == CONVERSATION_VARIABLE_NODE_ID: diff --git a/api/services/app_dsl_service.py b/api/services/app_dsl_service.py index f81ce8393e3a8a..15119247f887f0 100644 --- a/api/services/app_dsl_service.py +++ b/api/services/app_dsl_service.py @@ -1,7 +1,7 @@ import logging import uuid from enum import StrEnum -from typing import Optional, cast +from typing import Optional from urllib.parse import urlparse from uuid import uuid4 @@ -139,15 +139,6 @@ def import_app( status=ImportStatus.FAILED, error="Empty content from url", ) - - try: - content = cast(bytes, content).decode("utf-8") - except UnicodeDecodeError as e: - return Import( - id=import_id, - status=ImportStatus.FAILED, - error=f"Error decoding content: {e}", - ) except Exception as e: return Import( id=import_id, From cb3499166355d4b715a2fb5092d878581f7482ee Mon Sep 17 00:00:00 2001 From: -LAN- Date: Mon, 13 Jan 2025 15:55:16 +0800 Subject: [PATCH 37/42] fix: add type hints for App model and improve error handling in audio services (#12677) Signed-off-by: -LAN- --- api/controllers/console/app/audio.py | 8 ++++++-- api/services/audio_service.py | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/api/controllers/console/app/audio.py b/api/controllers/console/app/audio.py index 9d26af276d2fc3..12d9157dda5e2a 100644 --- a/api/controllers/console/app/audio.py +++ b/api/controllers/console/app/audio.py @@ -22,7 +22,7 @@ from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError from core.model_runtime.errors.invoke import InvokeError from libs.login import login_required -from models.model import AppMode +from models import App, AppMode from services.audio_service import AudioService from services.errors.audio import ( AudioTooLargeServiceError, @@ -79,7 +79,7 @@ class ChatMessageTextApi(Resource): @login_required @account_initialization_required @get_app_model - def post(self, app_model): + def post(self, app_model: App): from werkzeug.exceptions import InternalServerError try: @@ -98,9 +98,13 @@ def post(self, app_model): and app_model.workflow.features_dict ): text_to_speech = app_model.workflow.features_dict.get("text_to_speech") + if text_to_speech is None: + raise ValueError("TTS is not enabled") voice = args.get("voice") or text_to_speech.get("voice") else: try: + if app_model.app_model_config is None: + raise ValueError("AppModelConfig not found") voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice") except Exception: voice = None diff --git a/api/services/audio_service.py b/api/services/audio_service.py index f4178a69a4aada..294dfe4c8c3ceb 100644 --- a/api/services/audio_service.py +++ b/api/services/audio_service.py @@ -82,7 +82,7 @@ def transcript_tts( from app import app from extensions.ext_database import db - def invoke_tts(text_content: str, app_model, voice: Optional[str] = None): + def invoke_tts(text_content: str, app_model: App, voice: Optional[str] = None): with app.app_context(): if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}: workflow = app_model.workflow @@ -95,6 +95,8 @@ def invoke_tts(text_content: str, app_model, voice: Optional[str] = None): voice = features_dict["text_to_speech"].get("voice") if voice is None else voice else: + if app_model.app_model_config is None: + raise ValueError("AppModelConfig not found") text_to_speech_dict = app_model.app_model_config.text_to_speech_dict if not text_to_speech_dict.get("enabled"): From 69d58fbb50a0aca9ed42001db19c18a716b813ac Mon Sep 17 00:00:00 2001 From: Boris Feld Date: Mon, 13 Jan 2025 10:41:44 +0100 Subject: [PATCH 38/42] Add new integration with Opik Tracking tool (#11501) --- api/core/ops/entities/config_entity.py | 32 ++ api/core/ops/opik_trace/__init__.py | 0 api/core/ops/opik_trace/opik_trace.py | 469 ++++++++++++++++++ api/core/ops/ops_trace_manager.py | 8 + api/poetry.lock | 168 ++++++- api/pyproject.toml | 1 + api/services/ops_service.py | 11 +- .../[appId]/overview/tracing/config-popup.tsx | 76 ++- .../[appId]/overview/tracing/config.ts | 1 + .../[appId]/overview/tracing/panel.tsx | 27 +- .../tracing/provider-config-modal.tsx | 56 ++- .../overview/tracing/provider-panel.tsx | 3 +- .../[appId]/overview/tracing/type.ts | 8 + .../assets/public/tracing/opik-icon-big.svg | 87 ++++ .../icons/assets/public/tracing/opik-icon.svg | 88 ++++ .../icons/src/public/tracing/OpikIcon.json | 163 ++++++ .../icons/src/public/tracing/OpikIcon.tsx | 16 + .../icons/src/public/tracing/OpikIconBig.json | 162 ++++++ .../icons/src/public/tracing/OpikIconBig.tsx | 16 + .../base/icons/src/public/tracing/index.ts | 2 + web/i18n/en-US/app.ts | 4 + web/i18n/zh-Hans/app.ts | 4 + web/models/app.ts | 4 +- 23 files changed, 1380 insertions(+), 26 deletions(-) create mode 100644 api/core/ops/opik_trace/__init__.py create mode 100644 api/core/ops/opik_trace/opik_trace.py create mode 100644 web/app/components/base/icons/assets/public/tracing/opik-icon-big.svg create mode 100644 web/app/components/base/icons/assets/public/tracing/opik-icon.svg create mode 100644 web/app/components/base/icons/src/public/tracing/OpikIcon.json create mode 100644 web/app/components/base/icons/src/public/tracing/OpikIcon.tsx create mode 100644 web/app/components/base/icons/src/public/tracing/OpikIconBig.json create mode 100644 web/app/components/base/icons/src/public/tracing/OpikIconBig.tsx diff --git a/api/core/ops/entities/config_entity.py b/api/core/ops/entities/config_entity.py index ef0f9c708f14b0..b484242b611d79 100644 --- a/api/core/ops/entities/config_entity.py +++ b/api/core/ops/entities/config_entity.py @@ -6,6 +6,7 @@ class TracingProviderEnum(Enum): LANGFUSE = "langfuse" LANGSMITH = "langsmith" + OPIK = "opik" class BaseTracingConfig(BaseModel): @@ -56,5 +57,36 @@ def set_value(cls, v, info: ValidationInfo): return v +class OpikConfig(BaseTracingConfig): + """ + Model class for Opik tracing config. + """ + + api_key: str | None = None + project: str | None = None + workspace: str | None = None + url: str = "https://www.comet.com/opik/api/" + + @field_validator("project") + @classmethod + def project_validator(cls, v, info: ValidationInfo): + if v is None or v == "": + v = "Default Project" + + return v + + @field_validator("url") + @classmethod + def url_validator(cls, v, info: ValidationInfo): + if v is None or v == "": + v = "https://www.comet.com/opik/api/" + if not v.startswith(("https://", "http://")): + raise ValueError("url must start with https:// or http://") + if not v.endswith("/api/"): + raise ValueError("url should ends with /api/") + + return v + + OPS_FILE_PATH = "ops_trace/" OPS_TRACE_FAILED_KEY = "FAILED_OPS_TRACE" diff --git a/api/core/ops/opik_trace/__init__.py b/api/core/ops/opik_trace/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/api/core/ops/opik_trace/opik_trace.py b/api/core/ops/opik_trace/opik_trace.py new file mode 100644 index 00000000000000..fabf38fbd64178 --- /dev/null +++ b/api/core/ops/opik_trace/opik_trace.py @@ -0,0 +1,469 @@ +import json +import logging +import os +import uuid +from datetime import datetime, timedelta +from typing import Optional, cast + +from opik import Opik, Trace +from opik.id_helpers import uuid4_to_uuid7 + +from core.ops.base_trace_instance import BaseTraceInstance +from core.ops.entities.config_entity import OpikConfig +from core.ops.entities.trace_entity import ( + BaseTraceInfo, + DatasetRetrievalTraceInfo, + GenerateNameTraceInfo, + MessageTraceInfo, + ModerationTraceInfo, + SuggestedQuestionTraceInfo, + ToolTraceInfo, + TraceTaskName, + WorkflowTraceInfo, +) +from extensions.ext_database import db +from models.model import EndUser, MessageFile +from models.workflow import WorkflowNodeExecution + +logger = logging.getLogger(__name__) + + +def wrap_dict(key_name, data): + """Make sure that the input data is a dict""" + if not isinstance(data, dict): + return {key_name: data} + + return data + + +def wrap_metadata(metadata, **kwargs): + """Add common metatada to all Traces and Spans""" + metadata["created_from"] = "dify" + + metadata.update(kwargs) + + return metadata + + +def prepare_opik_uuid(user_datetime: Optional[datetime], user_uuid: Optional[str]): + """Opik needs UUIDv7 while Dify uses UUIDv4 for identifier of most + messages and objects. The type-hints of BaseTraceInfo indicates that + objects start_time and message_id could be null which means we cannot map + it to a UUIDv7. Given that we have no way to identify that object + uniquely, generate a new random one UUIDv7 in that case. + """ + + if user_datetime is None: + user_datetime = datetime.now() + + if user_uuid is None: + user_uuid = str(uuid.uuid4()) + + return uuid4_to_uuid7(user_datetime, user_uuid) + + +class OpikDataTrace(BaseTraceInstance): + def __init__( + self, + opik_config: OpikConfig, + ): + super().__init__(opik_config) + self.opik_client = Opik( + project_name=opik_config.project, + workspace=opik_config.workspace, + host=opik_config.url, + api_key=opik_config.api_key, + ) + self.project = opik_config.project + self.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001") + + def trace(self, trace_info: BaseTraceInfo): + if isinstance(trace_info, WorkflowTraceInfo): + self.workflow_trace(trace_info) + if isinstance(trace_info, MessageTraceInfo): + self.message_trace(trace_info) + if isinstance(trace_info, ModerationTraceInfo): + self.moderation_trace(trace_info) + if isinstance(trace_info, SuggestedQuestionTraceInfo): + self.suggested_question_trace(trace_info) + if isinstance(trace_info, DatasetRetrievalTraceInfo): + self.dataset_retrieval_trace(trace_info) + if isinstance(trace_info, ToolTraceInfo): + self.tool_trace(trace_info) + if isinstance(trace_info, GenerateNameTraceInfo): + self.generate_name_trace(trace_info) + + def workflow_trace(self, trace_info: WorkflowTraceInfo): + dify_trace_id = trace_info.workflow_run_id + opik_trace_id = prepare_opik_uuid(trace_info.start_time, dify_trace_id) + workflow_metadata = wrap_metadata( + trace_info.metadata, message_id=trace_info.message_id, workflow_app_log_id=trace_info.workflow_app_log_id + ) + root_span_id = None + + if trace_info.message_id: + dify_trace_id = trace_info.message_id + opik_trace_id = prepare_opik_uuid(trace_info.start_time, dify_trace_id) + + trace_data = { + "id": opik_trace_id, + "name": TraceTaskName.MESSAGE_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": workflow_metadata, + "input": wrap_dict("input", trace_info.workflow_run_inputs), + "output": wrap_dict("output", trace_info.workflow_run_outputs), + "tags": ["message", "workflow"], + "project_name": self.project, + } + self.add_trace(trace_data) + + root_span_id = prepare_opik_uuid(trace_info.start_time, trace_info.workflow_run_id) + span_data = { + "id": root_span_id, + "parent_span_id": None, + "trace_id": opik_trace_id, + "name": TraceTaskName.WORKFLOW_TRACE.value, + "input": wrap_dict("input", trace_info.workflow_run_inputs), + "output": wrap_dict("output", trace_info.workflow_run_outputs), + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": workflow_metadata, + "tags": ["workflow"], + "project_name": self.project, + } + self.add_span(span_data) + else: + trace_data = { + "id": opik_trace_id, + "name": TraceTaskName.MESSAGE_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": workflow_metadata, + "input": wrap_dict("input", trace_info.workflow_run_inputs), + "output": wrap_dict("output", trace_info.workflow_run_outputs), + "tags": ["workflow"], + "project_name": self.project, + } + self.add_trace(trace_data) + + # through workflow_run_id get all_nodes_execution + workflow_nodes_execution_id_records = ( + db.session.query(WorkflowNodeExecution.id) + .filter(WorkflowNodeExecution.workflow_run_id == trace_info.workflow_run_id) + .all() + ) + + for node_execution_id_record in workflow_nodes_execution_id_records: + node_execution = ( + db.session.query( + WorkflowNodeExecution.id, + WorkflowNodeExecution.tenant_id, + WorkflowNodeExecution.app_id, + WorkflowNodeExecution.title, + WorkflowNodeExecution.node_type, + WorkflowNodeExecution.status, + WorkflowNodeExecution.inputs, + WorkflowNodeExecution.outputs, + WorkflowNodeExecution.created_at, + WorkflowNodeExecution.elapsed_time, + WorkflowNodeExecution.process_data, + WorkflowNodeExecution.execution_metadata, + ) + .filter(WorkflowNodeExecution.id == node_execution_id_record.id) + .first() + ) + + if not node_execution: + continue + + node_execution_id = node_execution.id + tenant_id = node_execution.tenant_id + app_id = node_execution.app_id + node_name = node_execution.title + node_type = node_execution.node_type + status = node_execution.status + if node_type == "llm": + inputs = ( + json.loads(node_execution.process_data).get("prompts", {}) if node_execution.process_data else {} + ) + else: + inputs = json.loads(node_execution.inputs) if node_execution.inputs else {} + outputs = json.loads(node_execution.outputs) if node_execution.outputs else {} + created_at = node_execution.created_at or datetime.now() + elapsed_time = node_execution.elapsed_time + finished_at = created_at + timedelta(seconds=elapsed_time) + + execution_metadata = ( + json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {} + ) + metadata = execution_metadata.copy() + metadata.update( + { + "workflow_run_id": trace_info.workflow_run_id, + "node_execution_id": node_execution_id, + "tenant_id": tenant_id, + "app_id": app_id, + "app_name": node_name, + "node_type": node_type, + "status": status, + } + ) + + process_data = json.loads(node_execution.process_data) if node_execution.process_data else {} + + provider = None + model = None + total_tokens = 0 + completion_tokens = 0 + prompt_tokens = 0 + + if process_data and process_data.get("model_mode") == "chat": + run_type = "llm" + provider = process_data.get("model_provider", None) + model = process_data.get("model_name", "") + metadata.update( + { + "ls_provider": provider, + "ls_model_name": model, + } + ) + + try: + if outputs.get("usage"): + total_tokens = outputs["usage"].get("total_tokens", 0) + prompt_tokens = outputs["usage"].get("prompt_tokens", 0) + completion_tokens = outputs["usage"].get("completion_tokens", 0) + except Exception: + logger.error("Failed to extract usage", exc_info=True) + + else: + run_type = "tool" + + parent_span_id = trace_info.workflow_app_log_id or trace_info.workflow_run_id + + if not total_tokens: + total_tokens = execution_metadata.get("total_tokens", 0) + + span_data = { + "trace_id": opik_trace_id, + "id": prepare_opik_uuid(created_at, node_execution_id), + "parent_span_id": prepare_opik_uuid(trace_info.start_time, parent_span_id), + "name": node_type, + "type": run_type, + "start_time": created_at, + "end_time": finished_at, + "metadata": wrap_metadata(metadata), + "input": wrap_dict("input", inputs), + "output": wrap_dict("output", outputs), + "tags": ["node_execution"], + "project_name": self.project, + "usage": { + "total_tokens": total_tokens, + "completion_tokens": completion_tokens, + "prompt_tokens": prompt_tokens, + }, + "model": model, + "provider": provider, + } + + self.add_span(span_data) + + def message_trace(self, trace_info: MessageTraceInfo): + # get message file data + file_list = cast(list[str], trace_info.file_list) or [] + message_file_data: Optional[MessageFile] = trace_info.message_file_data + + if message_file_data is not None: + file_url = f"{self.file_base_url}/{message_file_data.url}" if message_file_data else "" + file_list.append(file_url) + + message_data = trace_info.message_data + if message_data is None: + return + + metadata = trace_info.metadata + message_id = trace_info.message_id + + user_id = message_data.from_account_id + metadata["user_id"] = user_id + metadata["file_list"] = file_list + + if message_data.from_end_user_id: + end_user_data: Optional[EndUser] = ( + db.session.query(EndUser).filter(EndUser.id == message_data.from_end_user_id).first() + ) + if end_user_data is not None: + end_user_id = end_user_data.session_id + metadata["end_user_id"] = end_user_id + + trace_data = { + "id": prepare_opik_uuid(trace_info.start_time, message_id), + "name": TraceTaskName.MESSAGE_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(metadata), + "input": trace_info.inputs, + "output": message_data.answer, + "tags": ["message", str(trace_info.conversation_mode)], + "project_name": self.project, + } + trace = self.add_trace(trace_data) + + span_data = { + "trace_id": trace.id, + "name": "llm", + "type": "llm", + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(metadata), + "input": {"input": trace_info.inputs}, + "output": {"output": message_data.answer}, + "tags": ["llm", str(trace_info.conversation_mode)], + "usage": { + "completion_tokens": trace_info.answer_tokens, + "prompt_tokens": trace_info.message_tokens, + "total_tokens": trace_info.total_tokens, + }, + "project_name": self.project, + } + self.add_span(span_data) + + def moderation_trace(self, trace_info: ModerationTraceInfo): + if trace_info.message_data is None: + return + + start_time = trace_info.start_time or trace_info.message_data.created_at + + span_data = { + "trace_id": prepare_opik_uuid(start_time, trace_info.message_id), + "name": TraceTaskName.MODERATION_TRACE.value, + "type": "tool", + "start_time": start_time, + "end_time": trace_info.end_time or trace_info.message_data.updated_at, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.inputs), + "output": { + "action": trace_info.action, + "flagged": trace_info.flagged, + "preset_response": trace_info.preset_response, + "inputs": trace_info.inputs, + }, + "tags": ["moderation"], + } + + self.add_span(span_data) + + def suggested_question_trace(self, trace_info: SuggestedQuestionTraceInfo): + message_data = trace_info.message_data + if message_data is None: + return + + start_time = trace_info.start_time or message_data.created_at + + span_data = { + "trace_id": prepare_opik_uuid(start_time, trace_info.message_id), + "name": TraceTaskName.SUGGESTED_QUESTION_TRACE.value, + "type": "tool", + "start_time": start_time, + "end_time": trace_info.end_time or message_data.updated_at, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.inputs), + "output": wrap_dict("output", trace_info.suggested_question), + "tags": ["suggested_question"], + } + + self.add_span(span_data) + + def dataset_retrieval_trace(self, trace_info: DatasetRetrievalTraceInfo): + if trace_info.message_data is None: + return + + start_time = trace_info.start_time or trace_info.message_data.created_at + + span_data = { + "trace_id": prepare_opik_uuid(start_time, trace_info.message_id), + "name": TraceTaskName.DATASET_RETRIEVAL_TRACE.value, + "type": "tool", + "start_time": start_time, + "end_time": trace_info.end_time or trace_info.message_data.updated_at, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.inputs), + "output": {"documents": trace_info.documents}, + "tags": ["dataset_retrieval"], + } + + self.add_span(span_data) + + def tool_trace(self, trace_info: ToolTraceInfo): + span_data = { + "trace_id": prepare_opik_uuid(trace_info.start_time, trace_info.message_id), + "name": trace_info.tool_name, + "type": "tool", + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.tool_inputs), + "output": wrap_dict("output", trace_info.tool_outputs), + "tags": ["tool", trace_info.tool_name], + } + + self.add_span(span_data) + + def generate_name_trace(self, trace_info: GenerateNameTraceInfo): + trace_data = { + "id": prepare_opik_uuid(trace_info.start_time, trace_info.message_id), + "name": TraceTaskName.GENERATE_NAME_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(trace_info.metadata), + "input": trace_info.inputs, + "output": trace_info.outputs, + "tags": ["generate_name"], + "project_name": self.project, + } + + trace = self.add_trace(trace_data) + + span_data = { + "trace_id": trace.id, + "name": TraceTaskName.GENERATE_NAME_TRACE.value, + "start_time": trace_info.start_time, + "end_time": trace_info.end_time, + "metadata": wrap_metadata(trace_info.metadata), + "input": wrap_dict("input", trace_info.inputs), + "output": wrap_dict("output", trace_info.outputs), + "tags": ["generate_name"], + } + + self.add_span(span_data) + + def add_trace(self, opik_trace_data: dict) -> Trace: + try: + trace = self.opik_client.trace(**opik_trace_data) + logger.debug("Opik Trace created successfully") + return trace + except Exception as e: + raise ValueError(f"Opik Failed to create trace: {str(e)}") + + def add_span(self, opik_span_data: dict): + try: + self.opik_client.span(**opik_span_data) + logger.debug("Opik Span created successfully") + except Exception as e: + raise ValueError(f"Opik Failed to create span: {str(e)}") + + def api_check(self): + try: + self.opik_client.auth_check() + return True + except Exception as e: + logger.info(f"Opik API check failed: {str(e)}", exc_info=True) + raise ValueError(f"Opik API check failed: {str(e)}") + + def get_project_url(self): + try: + return self.opik_client.get_project_url(project_name=self.project) + except Exception as e: + logger.info(f"Opik get run url failed: {str(e)}", exc_info=True) + raise ValueError(f"Opik get run url failed: {str(e)}") diff --git a/api/core/ops/ops_trace_manager.py b/api/core/ops/ops_trace_manager.py index 691cb8d400964c..c153e3f9dd6d47 100644 --- a/api/core/ops/ops_trace_manager.py +++ b/api/core/ops/ops_trace_manager.py @@ -17,6 +17,7 @@ OPS_FILE_PATH, LangfuseConfig, LangSmithConfig, + OpikConfig, TracingProviderEnum, ) from core.ops.entities.trace_entity import ( @@ -32,6 +33,7 @@ ) from core.ops.langfuse_trace.langfuse_trace import LangFuseDataTrace from core.ops.langsmith_trace.langsmith_trace import LangSmithDataTrace +from core.ops.opik_trace.opik_trace import OpikDataTrace from core.ops.utils import get_message_data from extensions.ext_database import db from extensions.ext_storage import storage @@ -52,6 +54,12 @@ "other_keys": ["project", "endpoint"], "trace_instance": LangSmithDataTrace, }, + TracingProviderEnum.OPIK.value: { + "config_class": OpikConfig, + "secret_keys": ["api_key"], + "other_keys": ["project", "url", "workspace"], + "trace_instance": OpikDataTrace, + }, } diff --git a/api/poetry.lock b/api/poetry.lock index fe80545e7ca682..7feb942a7f69d0 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "aiofiles" @@ -4693,6 +4693,134 @@ requests-toolbelt = ">=1.0.0,<2.0.0" [package.extras] langsmith-pyo3 = ["langsmith-pyo3 (>=0.1.0rc2,<0.2.0)"] +[[package]] +name = "levenshtein" +version = "0.26.1" +description = "Python extension for computing string edit distances and similarities." +optional = false +python-versions = ">=3.9" +files = [ + {file = "levenshtein-0.26.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:8dc4a4aecad538d944a1264c12769c99e3c0bf8e741fc5e454cc954913befb2e"}, + {file = "levenshtein-0.26.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ec108f368c12b25787c8b1a4537a1452bc53861c3ee4abc810cc74098278edcd"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69229d651c97ed5b55b7ce92481ed00635cdbb80fbfb282a22636e6945dc52d5"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:79dcd157046d62482a7719b08ba9e3ce9ed3fc5b015af8ea989c734c702aedd4"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6f53f9173ae21b650b4ed8aef1d0ad0c37821f367c221a982f4d2922b3044e0d"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f3956f3c5c229257dbeabe0b6aacd2c083ebcc1e335842a6ff2217fe6cc03b6b"}, + {file = "levenshtein-0.26.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e1e83af732726987d2c4cd736f415dae8b966ba17b7a2239c8b7ffe70bfb5543"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4f052c55046c2a9c9b5f742f39e02fa6e8db8039048b8c1c9e9fdd27c8a240a1"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:9895b3a98f6709e293615fde0dcd1bb0982364278fa2072361a1a31b3e388b7a"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:a3777de1d8bfca054465229beed23994f926311ce666f5a392c8859bb2722f16"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:81c57e1135c38c5e6e3675b5e2077d8a8d3be32bf0a46c57276c092b1dffc697"}, + {file = "levenshtein-0.26.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:91d5e7d984891df3eff7ea9fec8cf06fdfacc03cd074fd1a410435706f73b079"}, + {file = "levenshtein-0.26.1-cp310-cp310-win32.whl", hash = "sha256:f48abff54054b4142ad03b323e80aa89b1d15cabc48ff49eb7a6ff7621829a56"}, + {file = "levenshtein-0.26.1-cp310-cp310-win_amd64.whl", hash = "sha256:79dd6ad799784ea7b23edd56e3bf94b3ca866c4c6dee845658ee75bb4aefdabf"}, + {file = "levenshtein-0.26.1-cp310-cp310-win_arm64.whl", hash = "sha256:3351ddb105ef010cc2ce474894c5d213c83dddb7abb96400beaa4926b0b745bd"}, + {file = "levenshtein-0.26.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:44c51f5d33b3cfb9db518b36f1288437a509edd82da94c4400f6a681758e0cb6"}, + {file = "levenshtein-0.26.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56b93203e725f9df660e2afe3d26ba07d71871b6d6e05b8b767e688e23dfb076"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:270d36c5da04a0d89990660aea8542227cbd8f5bc34e9fdfadd34916ff904520"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:480674c05077eeb0b0f748546d4fcbb386d7c737f9fff0010400da3e8b552942"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:13946e37323728695ba7a22f3345c2e907d23f4600bc700bf9b4352fb0c72a48"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ceb673f572d1d0dc9b1cd75792bb8bad2ae8eb78a7c6721e23a3867d318cb6f2"}, + {file = "levenshtein-0.26.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42d6fa242e3b310ce6bfd5af0c83e65ef10b608b885b3bb69863c01fb2fcff98"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b8b68295808893a81e0a1dbc2274c30dd90880f14d23078e8eb4325ee615fc68"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:b01061d377d1944eb67bc40bef5d4d2f762c6ab01598efd9297ce5d0047eb1b5"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9d12c8390f156745e533d01b30773b9753e41d8bbf8bf9dac4b97628cdf16314"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:48825c9f967f922061329d1481b70e9fee937fc68322d6979bc623f69f75bc91"}, + {file = "levenshtein-0.26.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d8ec137170b95736842f99c0e7a9fd8f5641d0c1b63b08ce027198545d983e2b"}, + {file = "levenshtein-0.26.1-cp311-cp311-win32.whl", hash = "sha256:798f2b525a2e90562f1ba9da21010dde0d73730e277acaa5c52d2a6364fd3e2a"}, + {file = "levenshtein-0.26.1-cp311-cp311-win_amd64.whl", hash = "sha256:55b1024516c59df55f1cf1a8651659a568f2c5929d863d3da1ce8893753153bd"}, + {file = "levenshtein-0.26.1-cp311-cp311-win_arm64.whl", hash = "sha256:e52575cbc6b9764ea138a6f82d73d3b1bc685fe62e207ff46a963d4c773799f6"}, + {file = "levenshtein-0.26.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:cc741ca406d3704dc331a69c04b061fc952509a069b79cab8287413f434684bd"}, + {file = "levenshtein-0.26.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:821ace3b4e1c2e02b43cf5dc61aac2ea43bdb39837ac890919c225a2c3f2fea4"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92694c9396f55d4c91087efacf81297bef152893806fc54c289fc0254b45384"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:51ba374de7a1797d04a14a4f0ad3602d2d71fef4206bb20a6baaa6b6a502da58"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f7aa5c3327dda4ef952769bacec09c09ff5bf426e07fdc94478c37955681885b"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33e2517e8d3c221de2d1183f400aed64211fcfc77077b291ed9f3bb64f141cdc"}, + {file = "levenshtein-0.26.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9092b622765c7649dd1d8af0f43354723dd6f4e570ac079ffd90b41033957438"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:fc16796c85d7d8b259881d59cc8b5e22e940901928c2ff6924b2c967924e8a0b"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e4370733967f5994ceeed8dc211089bedd45832ee688cecea17bfd35a9eb22b9"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:3535ecfd88c9b283976b5bc61265855f59bba361881e92ed2b5367b6990c93fe"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:90236e93d98bdfd708883a6767826fafd976dac8af8fc4a0fb423d4fa08e1bf0"}, + {file = "levenshtein-0.26.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:04b7cabb82edf566b1579b3ed60aac0eec116655af75a3c551fee8754ffce2ea"}, + {file = "levenshtein-0.26.1-cp312-cp312-win32.whl", hash = "sha256:ae382af8c76f6d2a040c0d9ca978baf461702ceb3f79a0a3f6da8d596a484c5b"}, + {file = "levenshtein-0.26.1-cp312-cp312-win_amd64.whl", hash = "sha256:fd091209798cfdce53746f5769987b4108fe941c54fb2e058c016ffc47872918"}, + {file = "levenshtein-0.26.1-cp312-cp312-win_arm64.whl", hash = "sha256:7e82f2ea44a81ad6b30d92a110e04cd3c8c7c6034b629aca30a3067fa174ae89"}, + {file = "levenshtein-0.26.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:790374a9f5d2cbdb30ee780403a62e59bef51453ac020668c1564d1e43438f0e"}, + {file = "levenshtein-0.26.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7b05c0415c386d00efda83d48db9db68edd02878d6dbc6df01194f12062be1bb"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3114586032361722ddededf28401ce5baf1cf617f9f49fb86b8766a45a423ff"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2532f8a13b68bf09f152d906f118a88da2063da22f44c90e904b142b0a53d534"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:219c30be6aa734bf927188d1208b7d78d202a3eb017b1c5f01ab2034d2d4ccca"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:397e245e77f87836308bd56305bba630010cd8298c34c4c44bd94990cdb3b7b1"}, + {file = "levenshtein-0.26.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aeff6ea3576f72e26901544c6c55c72a7b79b9983b6f913cba0e9edbf2f87a97"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a19862e3539a697df722a08793994e334cd12791e8144851e8a1dee95a17ff63"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:dc3b5a64f57c3c078d58b1e447f7d68cad7ae1b23abe689215d03fc434f8f176"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bb6c7347424a91317c5e1b68041677e4c8ed3e7823b5bbaedb95bffb3c3497ea"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:b817376de4195a207cc0e4ca37754c0e1e1078c2a2d35a6ae502afde87212f9e"}, + {file = "levenshtein-0.26.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7b50c3620ff47c9887debbb4c154aaaac3e46be7fc2e5789ee8dbe128bce6a17"}, + {file = "levenshtein-0.26.1-cp313-cp313-win32.whl", hash = "sha256:9fb859da90262eb474c190b3ca1e61dee83add022c676520f5c05fdd60df902a"}, + {file = "levenshtein-0.26.1-cp313-cp313-win_amd64.whl", hash = "sha256:8adcc90e3a5bfb0a463581d85e599d950fe3c2938ac6247b29388b64997f6e2d"}, + {file = "levenshtein-0.26.1-cp313-cp313-win_arm64.whl", hash = "sha256:c2599407e029865dc66d210b8804c7768cbdbf60f061d993bb488d5242b0b73e"}, + {file = "levenshtein-0.26.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:dc54ced948fc3feafce8ad4ba4239d8ffc733a0d70e40c0363ac2a7ab2b7251e"}, + {file = "levenshtein-0.26.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e6516f69213ae393a220e904332f1a6bfc299ba22cf27a6520a1663a08eba0fb"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4cfea4eada1746d0c75a864bc7e9e63d4a6e987c852d6cec8d9cb0c83afe25b"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a323161dfeeac6800eb13cfe76a8194aec589cd948bcf1cdc03f66cc3ec26b72"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2c23e749b68ebc9a20b9047317b5cd2053b5856315bc8636037a8adcbb98bed1"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f80dd7432d4b6cf493d012d22148db7af769017deb31273e43406b1fb7f091c"}, + {file = "levenshtein-0.26.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0ae7cd6e4312c6ef34b2e273836d18f9fff518d84d823feff5ad7c49668256e0"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dcdad740e841d791b805421c2b20e859b4ed556396d3063b3aa64cd055be648c"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:e07afb1613d6f5fd99abd4e53ad3b446b4efaa0f0d8e9dfb1d6d1b9f3f884d32"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:f1add8f1d83099a98ae4ac472d896b7e36db48c39d3db25adf12b373823cdeff"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:1010814b1d7a60833a951f2756dfc5c10b61d09976ce96a0edae8fecdfb0ea7c"}, + {file = "levenshtein-0.26.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:33fa329d1bb65ce85e83ceda281aea31cee9f2f6e167092cea54f922080bcc66"}, + {file = "levenshtein-0.26.1-cp39-cp39-win32.whl", hash = "sha256:488a945312f2f16460ab61df5b4beb1ea2254c521668fd142ce6298006296c98"}, + {file = "levenshtein-0.26.1-cp39-cp39-win_amd64.whl", hash = "sha256:9f942104adfddd4b336c3997050121328c39479f69de702d7d144abb69ea7ab9"}, + {file = "levenshtein-0.26.1-cp39-cp39-win_arm64.whl", hash = "sha256:c1d8f85b2672939f85086ed75effcf768f6077516a3e299c2ba1f91bc4644c22"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:6cf8f1efaf90ca585640c5d418c30b7d66d9ac215cee114593957161f63acde0"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d5b2953978b8c158dd5cd93af8216a5cfddbf9de66cf5481c2955f44bb20767a"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b952b3732c4631c49917d4b15d78cb4a2aa006c1d5c12e2a23ba8e18a307a055"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:07227281e12071168e6ae59238918a56d2a0682e529f747b5431664f302c0b42"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8191241cd8934feaf4d05d0cc0e5e72877cbb17c53bbf8c92af9f1aedaa247e9"}, + {file = "levenshtein-0.26.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:9e70d7ee157a9b698c73014f6e2b160830e7d2d64d2e342fefc3079af3c356fc"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0eb3059f826f6cb0a5bca4a85928070f01e8202e7ccafcba94453470f83e49d4"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:6c389e44da12d6fb1d7ba0a709a32a96c9391e9be4160ccb9269f37e040599ee"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e9de292f2c51a7d34a0ae23bec05391b8f61f35781cd3e4c6d0533e06250c55"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d87215113259efdca8716e53b6d59ab6d6009e119d95d45eccc083148855f33"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:18f00a3eebf68a82fb651d8d0e810c10bfaa60c555d21dde3ff81350c74fb4c2"}, + {file = "levenshtein-0.26.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:b3554c1b59de63d05075577380340c185ff41b028e541c0888fddab3c259a2b4"}, + {file = "levenshtein-0.26.1.tar.gz", hash = "sha256:0d19ba22330d50609b2349021ec3cf7d905c6fe21195a2d0d876a146e7ed2575"}, +] + +[package.dependencies] +rapidfuzz = ">=3.9.0,<4.0.0" + +[[package]] +name = "litellm" +version = "1.51.3" +description = "Library to easily interface with LLM API providers" +optional = false +python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" +files = [ + {file = "litellm-1.51.3-py3-none-any.whl", hash = "sha256:440d3c7cc5ab8eeb12cee8f4d806bff05b7db834ebc11117d7fa070a1142ced5"}, + {file = "litellm-1.51.3.tar.gz", hash = "sha256:31eff9fcbf7b058bac0fd7432c4ea0487e8555f12446a1f30e5862e33716f44d"}, +] + +[package.dependencies] +aiohttp = "*" +click = "*" +importlib-metadata = ">=6.8.0" +jinja2 = ">=3.1.2,<4.0.0" +jsonschema = ">=4.22.0,<5.0.0" +openai = ">=1.52.0" +pydantic = ">=2.0.0,<3.0.0" +python-dotenv = ">=0.2.0" +requests = ">=2.31.0,<3.0.0" +tiktoken = ">=0.7.0" +tokenizers = "*" + +[package.extras] +extra-proxy = ["azure-identity (>=1.15.0,<2.0.0)", "azure-keyvault-secrets (>=4.8.0,<5.0.0)", "google-cloud-kms (>=2.21.3,<3.0.0)", "prisma (==0.11.0)", "resend (>=0.8.0,<0.9.0)"] +proxy = ["PyJWT (>=2.8.0,<3.0.0)", "apscheduler (>=3.10.4,<4.0.0)", "backoff", "cryptography (>=42.0.5,<43.0.0)", "fastapi (>=0.111.0,<0.112.0)", "fastapi-sso (>=0.10.0,<0.11.0)", "gunicorn (>=22.0.0,<23.0.0)", "orjson (>=3.9.7,<4.0.0)", "pynacl (>=1.5.0,<2.0.0)", "python-multipart (>=0.0.9,<0.0.10)", "pyyaml (>=6.0.1,<7.0.0)", "rq", "uvicorn (>=0.22.0,<0.23.0)"] + [[package]] name = "llvmlite" version = "0.43.0" @@ -6265,6 +6393,31 @@ files = [ {file = "opentelemetry_util_http-0.50b0.tar.gz", hash = "sha256:dc4606027e1bc02aabb9533cc330dd43f874fca492e4175c31d7154f341754af"}, ] +[[package]] +name = "opik" +version = "1.3.4" +description = "Comet tool for logging and evaluating LLM traces" +optional = false +python-versions = ">=3.8" +files = [ + {file = "opik-1.3.4-py3-none-any.whl", hash = "sha256:c5e10a9f1fb18188471cce2ae8b841e8b187d04ee3b1aed01c643102bae588fb"}, + {file = "opik-1.3.4.tar.gz", hash = "sha256:6013d3af4aea61f38b9e7121aa5d8cf4305a5ed3807b3f43d9ab91602b2a5785"}, +] + +[package.dependencies] +click = "*" +httpx = "<0.28.0" +levenshtein = "<1.0.0" +litellm = "*" +openai = "<2.0.0" +pydantic = ">=2.0.0,<3.0.0" +pydantic-settings = ">=2.0.0,<3.0.0" +pytest = "*" +rich = "*" +tenacity = "*" +tqdm = "*" +uuid6 = "*" + [[package]] name = "oracledb" version = "2.2.1" @@ -10230,6 +10383,17 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "uuid6" +version = "2024.7.10" +description = "New time-based UUID formats which are suited for use as a database key" +optional = false +python-versions = ">=3.8" +files = [ + {file = "uuid6-2024.7.10-py3-none-any.whl", hash = "sha256:93432c00ba403751f722829ad21759ff9db051dea140bf81493271e8e4dd18b7"}, + {file = "uuid6-2024.7.10.tar.gz", hash = "sha256:2d29d7f63f593caaeea0e0d0dd0ad8129c9c663b29e19bdf882e864bedf18fb0"}, +] + [[package]] name = "uvicorn" version = "0.34.0" @@ -11220,4 +11384,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = ">=3.11,<3.13" -content-hash = "907718f7ca775ad226c1f668f4bb6c6dbfa6cacc556fce43a8ad0b6f3c35095a" +content-hash = "3bb0ce64c87712cf105c75105a0ca75c0523d6b27001ff6a623bb2a0d1343003" diff --git a/api/pyproject.toml b/api/pyproject.toml index f8c6f599d182b0..6e2ba4cdb48325 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -59,6 +59,7 @@ numpy = "~1.26.4" oci = "~2.135.1" openai = "~1.52.0" openpyxl = "~3.1.5" +opik = "~1.3.4" pandas = { version = "~2.2.2", extras = ["performance", "excel"] } pandas-stubs = "~2.2.3.241009" psycogreen = "~1.0.2" diff --git a/api/services/ops_service.py b/api/services/ops_service.py index fc1e08518b1945..78340d2bcc2cee 100644 --- a/api/services/ops_service.py +++ b/api/services/ops_service.py @@ -59,6 +59,15 @@ def get_tracing_app_config(cls, app_id: str, tracing_provider: str): except Exception: new_decrypt_tracing_config.update({"project_url": "https://smith.langchain.com/"}) + if tracing_provider == "opik" and ( + "project_url" not in decrypt_tracing_config or not decrypt_tracing_config.get("project_url") + ): + try: + project_url = OpsTraceManager.get_trace_config_project_url(decrypt_tracing_config, tracing_provider) + new_decrypt_tracing_config.update({"project_url": project_url}) + except Exception: + new_decrypt_tracing_config.update({"project_url": "https://www.comet.com/opik/"}) + trace_config_data.tracing_config = new_decrypt_tracing_config return trace_config_data.to_dict() @@ -92,7 +101,7 @@ def create_tracing_app_config(cls, app_id: str, tracing_provider: str, tracing_c if tracing_provider == "langfuse": project_key = OpsTraceManager.get_trace_config_project_key(tracing_config, tracing_provider) project_url = "{host}/project/{key}".format(host=tracing_config.get("host"), key=project_key) - elif tracing_provider == "langsmith": + elif tracing_provider in ("langsmith", "opik"): project_url = OpsTraceManager.get_trace_config_project_url(tracing_config, tracing_provider) else: project_url = None diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx index 8e3d8f9ec6583f..17f46c258d4ede 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/config-popup.tsx @@ -5,7 +5,7 @@ import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' import TracingIcon from './tracing-icon' import ProviderPanel from './provider-panel' -import type { LangFuseConfig, LangSmithConfig } from './type' +import type { LangFuseConfig, LangSmithConfig, OpikConfig } from './type' import { TracingProvider } from './type' import ProviderConfigModal from './provider-config-modal' import Indicator from '@/app/components/header/indicator' @@ -23,7 +23,8 @@ export type PopupProps = { onChooseProvider: (provider: TracingProvider) => void langSmithConfig: LangSmithConfig | null langFuseConfig: LangFuseConfig | null - onConfigUpdated: (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig) => void + opikConfig: OpikConfig | null + onConfigUpdated: (provider: TracingProvider, payload: LangSmithConfig | LangFuseConfig | OpikConfig) => void onConfigRemoved: (provider: TracingProvider) => void } @@ -36,6 +37,7 @@ const ConfigPopup: FC = ({ onChooseProvider, langSmithConfig, langFuseConfig, + opikConfig, onConfigUpdated, onConfigRemoved, }) => { @@ -59,7 +61,7 @@ const ConfigPopup: FC = ({ } }, [onChooseProvider]) - const handleConfigUpdated = useCallback((payload: LangSmithConfig | LangFuseConfig) => { + const handleConfigUpdated = useCallback((payload: LangSmithConfig | LangFuseConfig | OpikConfig) => { onConfigUpdated(currentProvider!, payload) hideConfigModal() }, [currentProvider, hideConfigModal, onConfigUpdated]) @@ -69,8 +71,8 @@ const ConfigPopup: FC = ({ hideConfigModal() }, [currentProvider, hideConfigModal, onConfigRemoved]) - const providerAllConfigured = langSmithConfig && langFuseConfig - const providerAllNotConfigured = !langSmithConfig && !langFuseConfig + const providerAllConfigured = langSmithConfig && langFuseConfig && opikConfig + const providerAllNotConfigured = !langSmithConfig && !langFuseConfig && !opikConfig const switchContent = ( = ({ onConfig={handleOnConfig(TracingProvider.langSmith)} isChosen={chosenProvider === TracingProvider.langSmith} onChoose={handleOnChoose(TracingProvider.langSmith)} + key="langSmith-provider-panel" /> ) @@ -102,9 +105,61 @@ const ConfigPopup: FC = ({ onConfig={handleOnConfig(TracingProvider.langfuse)} isChosen={chosenProvider === TracingProvider.langfuse} onChoose={handleOnChoose(TracingProvider.langfuse)} + key="langfuse-provider-panel" /> ) + const opikPanel = ( + + ) + + const configuredProviderPanel = () => { + const configuredPanels: ProviderPanel[] = [] + + if (langSmithConfig) + configuredPanels.push(langSmithPanel) + + if (langFuseConfig) + configuredPanels.push(langfusePanel) + + if (opikConfig) + configuredPanels.push(opikPanel) + + return configuredPanels + } + + const moreProviderPanel = () => { + const notConfiguredPanels: ProviderPanel[] = [] + + if (!langSmithConfig) + notConfiguredPanels.push(langSmithPanel) + + if (!langFuseConfig) + notConfiguredPanels.push(langfusePanel) + + if (!opikConfig) + notConfiguredPanels.push(opikPanel) + + return notConfiguredPanels + } + + const configuredProviderConfig = () => { + if (currentProvider === TracingProvider.langSmith) + return langSmithConfig + if (currentProvider === TracingProvider.langfuse) + return langFuseConfig + return opikConfig + } + return (
    @@ -146,18 +201,19 @@ const ConfigPopup: FC = ({
    {langSmithPanel} {langfusePanel} + {opikPanel}
    ) : ( <>
    {t(`${I18N_PREFIX}.configProviderTitle.configured`)}
    -
    - {langSmithConfig ? langSmithPanel : langfusePanel} +
    + {configuredProviderPanel()}
    {t(`${I18N_PREFIX}.configProviderTitle.moreProvider`)}
    -
    - {!langSmithConfig ? langSmithPanel : langfusePanel} +
    + {moreProviderPanel()}
    )} @@ -167,7 +223,7 @@ const ConfigPopup: FC = ({ { }) } const inUseTracingProvider: TracingProvider | null = tracingStatus?.tracing_provider || null - const InUseProviderIcon = inUseTracingProvider === TracingProvider.langSmith ? LangsmithIcon : LangfuseIcon + + const InUseProviderIcon + = inUseTracingProvider === TracingProvider.langSmith + ? LangsmithIcon + : inUseTracingProvider === TracingProvider.langfuse + ? LangfuseIcon + : inUseTracingProvider === TracingProvider.opik + ? OpikIcon + : null const [langSmithConfig, setLangSmithConfig] = useState(null) const [langFuseConfig, setLangFuseConfig] = useState(null) - const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig) + const [opikConfig, setOpikConfig] = useState(null) + const hasConfiguredTracing = !!(langSmithConfig || langFuseConfig || opikConfig) const fetchTracingConfig = async () => { const { tracing_config: langSmithConfig, has_not_configured: langSmithHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langSmith }) @@ -83,6 +92,9 @@ const Panel: FC = () => { const { tracing_config: langFuseConfig, has_not_configured: langFuseHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.langfuse }) if (!langFuseHasNotConfig) setLangFuseConfig(langFuseConfig as LangFuseConfig) + const { tracing_config: opikConfig, has_not_configured: OpikHasNotConfig } = await doFetchTracingConfig({ appId, provider: TracingProvider.opik }) + if (!OpikHasNotConfig) + setOpikConfig(opikConfig as OpikConfig) } const handleTracingConfigUpdated = async (provider: TracingProvider) => { @@ -90,15 +102,19 @@ const Panel: FC = () => { const { tracing_config } = await doFetchTracingConfig({ appId, provider }) if (provider === TracingProvider.langSmith) setLangSmithConfig(tracing_config as LangSmithConfig) - else + else if (provider === TracingProvider.langSmith) setLangFuseConfig(tracing_config as LangFuseConfig) + else if (provider === TracingProvider.opik) + setOpikConfig(tracing_config as OpikConfig) } const handleTracingConfigRemoved = (provider: TracingProvider) => { if (provider === TracingProvider.langSmith) setLangSmithConfig(null) - else + else if (provider === TracingProvider.langSmith) setLangFuseConfig(null) + else if (provider === TracingProvider.opik) + setOpikConfig(null) if (provider === inUseTracingProvider) { handleTracingStatusChange({ enabled: false, @@ -167,6 +183,7 @@ const Panel: FC = () => { onChooseProvider={handleChooseProvider} langSmithConfig={langSmithConfig} langFuseConfig={langFuseConfig} + opikConfig={opikConfig} onConfigUpdated={handleTracingConfigUpdated} onConfigRemoved={handleTracingConfigRemoved} controlShowPopup={controlShowPopup} diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx index e7ecd2f4ce3d64..b813e9b1344109 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-config-modal.tsx @@ -4,7 +4,7 @@ import React, { useCallback, useState } from 'react' import { useTranslation } from 'react-i18next' import { useBoolean } from 'ahooks' import Field from './field' -import type { LangFuseConfig, LangSmithConfig } from './type' +import type { LangFuseConfig, LangSmithConfig, OpikConfig } from './type' import { TracingProvider } from './type' import { docURL } from './config' import { @@ -21,10 +21,10 @@ import Toast from '@/app/components/base/toast' type Props = { appId: string type: TracingProvider - payload?: LangSmithConfig | LangFuseConfig | null + payload?: LangSmithConfig | LangFuseConfig | OpikConfig | null onRemoved: () => void onCancel: () => void - onSaved: (payload: LangSmithConfig | LangFuseConfig) => void + onSaved: (payload: LangSmithConfig | LangFuseConfig | OpikConfig) => void onChosen: (provider: TracingProvider) => void } @@ -42,6 +42,13 @@ const langFuseConfigTemplate = { host: '', } +const opikConfigTemplate = { + api_key: '', + project: '', + url: '', + workspace: '', +} + const ProviderConfigModal: FC = ({ appId, type, @@ -55,14 +62,17 @@ const ProviderConfigModal: FC = ({ const isEdit = !!payload const isAdd = !isEdit const [isSaving, setIsSaving] = useState(false) - const [config, setConfig] = useState((() => { + const [config, setConfig] = useState((() => { if (isEdit) return payload if (type === TracingProvider.langSmith) return langSmithConfigTemplate - return langFuseConfigTemplate + else if (type === TracingProvider.langfuse) + return langFuseConfigTemplate + + return opikConfigTemplate })()) const [isShowRemoveConfirm, { setTrue: showRemoveConfirm, @@ -111,6 +121,10 @@ const ProviderConfigModal: FC = ({ errorMessage = t('common.errorMsg.fieldRequired', { field: 'Host' }) } + if (type === TracingProvider.opik) { + const postData = config as OpikConfig + } + return errorMessage }, [config, t, type]) const handleSave = useCallback(async () => { @@ -215,6 +229,38 @@ const ProviderConfigModal: FC = ({ /> )} + {type === TracingProvider.opik && ( + <> + + + + + + )}
    diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx index 6e5046ecf80e91..34e5bbeb0fbb06 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/provider-panel.tsx @@ -4,7 +4,7 @@ import React, { useCallback } from 'react' import { useTranslation } from 'react-i18next' import { TracingProvider } from './type' import cn from '@/utils/classnames' -import { LangfuseIconBig, LangsmithIconBig } from '@/app/components/base/icons/src/public/tracing' +import { LangfuseIconBig, LangsmithIconBig, OpikIconBig } from '@/app/components/base/icons/src/public/tracing' import { Settings04 } from '@/app/components/base/icons/src/vender/line/general' import { Eye as View } from '@/app/components/base/icons/src/vender/solid/general' @@ -24,6 +24,7 @@ const getIcon = (type: TracingProvider) => { return ({ [TracingProvider.langSmith]: LangsmithIconBig, [TracingProvider.langfuse]: LangfuseIconBig, + [TracingProvider.opik]: OpikIconBig, })[type] } diff --git a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts index e07cf37c9d26a4..982d01ffb39afd 100644 --- a/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts +++ b/web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type.ts @@ -1,6 +1,7 @@ export enum TracingProvider { langSmith = 'langsmith', langfuse = 'langfuse', + opik = 'opik', } export type LangSmithConfig = { @@ -14,3 +15,10 @@ export type LangFuseConfig = { secret_key: string host: string } + +export type OpikConfig = { + api_key: string + project: string + workspace: string + url: string +} diff --git a/web/app/components/base/icons/assets/public/tracing/opik-icon-big.svg b/web/app/components/base/icons/assets/public/tracing/opik-icon-big.svg new file mode 100644 index 00000000000000..99aa2da1a46478 --- /dev/null +++ b/web/app/components/base/icons/assets/public/tracing/opik-icon-big.svg @@ -0,0 +1,87 @@ + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/assets/public/tracing/opik-icon.svg b/web/app/components/base/icons/assets/public/tracing/opik-icon.svg new file mode 100644 index 00000000000000..fb4925486203e0 --- /dev/null +++ b/web/app/components/base/icons/assets/public/tracing/opik-icon.svg @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + diff --git a/web/app/components/base/icons/src/public/tracing/OpikIcon.json b/web/app/components/base/icons/src/public/tracing/OpikIcon.json new file mode 100644 index 00000000000000..5bab796c7888a9 --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/OpikIcon.json @@ -0,0 +1,163 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "47.133904", + "height": "16", + "viewBox": "0 0 47.133904 16", + "fill": "none", + "version": "1.1", + "id": "svg6", + "sodipodi:docname": "opik-icon.svg", + "inkscape:version": "1.3.2 (091e20ef0f, 2023-11-25)", + "xmlns:inkscape": "http://www.inkscape.org/namespaces/inkscape", + "xmlns:sodipodi": "http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd", + "xmlns": "http://www.w3.org/2000/svg", + "xmlns:svg": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "sodipodi:namedview", + "attributes": { + "id": "namedview6", + "pagecolor": "#b95d5d", + "bordercolor": "#666666", + "borderopacity": "1.0", + "inkscape:showpageshadow": "2", + "inkscape:pageopacity": "0.0", + "inkscape:pagecheckerboard": "0", + "inkscape:deskcolor": "#d1d1d1", + "inkscape:zoom": "18.615087", + "inkscape:cx": "34.541874", + "inkscape:cy": "18.882533", + "inkscape:window-width": "2560", + "inkscape:window-height": "1371", + "inkscape:window-x": "0", + "inkscape:window-y": "0", + "inkscape:window-maximized": "1", + "inkscape:current-layer": "svg6" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "width": "47.119099", + "height": "15.98219", + "fill": "#ffffff", + "id": "rect1", + "x": "0", + "y": "0", + "style": "stroke-width:0.0455515;fill:none", + "inkscape:label": "rect1" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M 9.6391824,3.9256084 C 7.4584431,2.9677242 4.8694901,3.9910489 3.8733677,6.2551834 2.8772499,8.519327 3.8730396,11.117275 6.0537561,12.075159 c 0.8795869,0.38635 1.8205107,0.450735 2.6986394,0.243012 0.3780009,-0.08943 0.7570043,0.144295 0.8464576,0.521993 0.089499,0.377699 -0.1443649,0.7564 -0.5224113,0.845827 C 7.9135024,13.961058 6.6590133,13.876457 5.4876434,13.361931 2.568556,12.079712 1.2888532,8.636894 2.5855671,5.6895232 3.8822857,2.7421295 7.286235,1.3566284 10.205295,2.6388372 11.986296,3.4211403 13.15908,5.0124429 13.505682,6.7988966 13.579642,7.1799648 13.330421,7.548739 12.949049,7.6226396 12.567721,7.6964947 12.198606,7.447473 12.124692,7.0664048 11.860478,5.7048679 10.972279,4.5111667 9.6391824,3.9256084 Z m 2.3662996,7.7665706 c 0.136116,0.570532 -0.216457,1.14325 -0.787445,1.279258 -0.570989,0.135962 -1.14421,-0.216283 -1.2802814,-0.786816 -0.1361171,-0.570532 0.2164564,-1.143295 0.7874454,-1.279258 0.570987,-0.136008 1.14421,0.216283 1.280281,0.786816 z m 0.885967,-0.810128 c 0.762836,-0.181679 1.233846,-0.9468664 1.052022,-1.7090479 -0.181824,-0.7622275 -0.947622,-1.2328598 -1.710414,-1.0511819 -0.762838,0.1816779 -1.233846,0.9468196 -1.052023,1.7090471 0.181823,0.7622277 0.947623,1.2328597 1.710415,1.0511827 z", + "fill": "url(#paint0_linear_3874_31725)", + "id": "path1", + "style": "fill:url(#paint0_linear_3874_31725);stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 20.686606,11.857146 c -0.721642,0 -1.364084,-0.153857 -1.927326,-0.461616 -0.563197,-0.316594 -1.007638,-0.747475 -1.333234,-1.292646 -0.325641,-0.54517 -0.488416,-1.165106 -0.488416,-1.8598076 0,-0.703444 0.162775,-1.32338 0.488416,-1.8598079 0.325596,-0.5451702 0.770037,-0.9716351 1.333234,-1.2794403 0.563242,-0.3077596 1.205684,-0.4616167 1.927326,-0.4616167 0.730437,0 1.377254,0.1538571 1.940495,0.4616167 0.571992,0.3078052 1.016434,0.7298533 1.333234,1.2662813 0.325641,0.5363823 0.488417,1.1606894 0.488417,1.8729669 0,0.6947016 -0.162776,1.3146376 -0.488417,1.8598076 -0.3168,0.545171 -0.761242,0.976052 -1.333234,1.292646 -0.563241,0.307759 -1.210058,0.461616 -1.940495,0.461616 z m 0,-1.411304 c 0.404796,0 0.765617,-0.08797 1.082418,-0.263821 0.316846,-0.17585 0.563242,-0.4308815 0.739232,-0.7650049 0.184831,-0.3341689 0.277246,-0.7254822 0.277246,-1.1739397 0,-0.4572454 -0.09241,-0.8485586 -0.277246,-1.1738941 C 22.332266,6.7350133 22.08587,6.4800268 21.769024,6.3041317 21.452223,6.1282821 21.095822,6.0403572 20.699776,6.0403572 c -0.404796,0 -0.765617,0.087925 -1.082418,0.2637745 -0.308051,0.1758951 -0.554446,0.4308816 -0.739277,0.7650506 -0.184786,0.3253355 -0.277201,0.7166487 -0.277201,1.1738941 0,0.4484575 0.09241,0.8397708 0.277201,1.1739397 0.184831,0.3341234 0.431226,0.5891549 0.739277,0.7650049 0.316801,0.17585 0.673201,0.263821 1.069248,0.263821 z", + "fill": "#3a3a3a", + "id": "path2", + "style": "stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 29.934026,11.857146 c -0.571992,0 -1.095634,-0.131865 -1.57088,-0.395684 -0.466407,-0.26382 -0.840442,-0.659504 -1.122018,-1.1871 -0.272826,-0.5364272 -0.409217,-1.2135074 -0.409217,-2.0312856 0,-0.826566 0.131971,-1.5036463 0.396002,-2.0312862 0.272826,-0.5275944 0.642442,-0.9189077 1.108848,-1.1738942 0.466406,-0.26382 0.998843,-0.3956845 1.597265,-0.3956845 0.695257,0 1.306893,0.1494859 1.83491,0.4484576 0.536811,0.2989717 0.959243,0.7166486 1.267248,1.253031 0.316801,0.5364279 0.475202,1.169523 0.475202,1.8993763 0,0.7298534 -0.158401,1.3673652 -0.475202,1.9125806 -0.308005,0.536383 -0.730437,0.95406 -1.267248,1.253031 -0.528017,0.298972 -1.139653,0.448458 -1.83491,0.448458 z m -3.142079,2.466539 c -0.422659,0 -0.765298,-0.342319 -0.765298,-0.764641 V 5.4859892 c 0,-0.4223213 0.342639,-0.7646408 0.765298,-0.7646408 h 0.04028 c 0.42266,0 0.765299,0.3423195 0.765299,0.7646408 v 0.8972793 l -0.05281,1.8730126 0.132016,1.8729669 v 3.429796 c 0,0.422322 -0.342639,0.764641 -0.765298,0.764641 z m 2.957293,-3.877843 c 0.396001,0 0.748027,-0.08797 1.056033,-0.263821 0.316801,-0.17585 0.567616,-0.4308815 0.752447,-0.7650049 0.184786,-0.3341689 0.277201,-0.7254822 0.277201,-1.1739397 0,-0.4572454 -0.09241,-0.8485586 -0.277201,-1.1738941 C 31.372889,6.7350133 31.122074,6.4800268 30.805273,6.3041317 30.497267,6.1282821 30.145241,6.0403572 29.74924,6.0403572 c -0.396046,0 -0.752447,0.087925 -1.069248,0.2637745 -0.316846,0.1758951 -0.567662,0.4308816 -0.752447,0.7650506 -0.184831,0.3253355 -0.277201,0.7166487 -0.277201,1.1738941 0,0.4484575 0.09237,0.8397708 0.277201,1.1739397 0.184785,0.3341234 0.435601,0.5891549 0.752447,0.7650049 0.316801,0.17585 0.673202,0.263821 1.069248,0.263821 z", + "fill": "#3a3a3a", + "id": "path3", + "style": "stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 35.841594,11.76485 c -0.422659,0 -0.765298,-0.342365 -0.765298,-0.764686 V 5.4859892 c 0,-0.4223213 0.342639,-0.7646408 0.765298,-0.7646408 h 0.119484 c 0.422659,0 0.765298,0.3423195 0.765298,0.7646408 v 5.5141748 c 0,0.422321 -0.342639,0.764686 -0.765298,0.764686 z m 0.06635,-8.2042457 c -0.308006,0 -0.563241,-0.096726 -0.765662,-0.2901837 -0.19358,-0.1934528 -0.290371,-0.4264787 -0.290371,-0.6990729 0,-0.2813867 0.0968,-0.5144125 0.290371,-0.6990728 0.202421,-0.1934528 0.457656,-0.2901838 0.765662,-0.2901838 0.308006,0 0.558822,0.092332 0.752448,0.2769928 0.202375,0.1758678 0.303585,0.4001011 0.303585,0.6726954 0,0.2901792 -0.0968,0.536396 -0.290415,0.7386413 -0.19358,0.1934573 -0.448817,0.2901837 -0.765618,0.2901837 z", + "fill": "#3a3a3a", + "id": "path4", + "style": "stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 40.238571,10.195226 0.0396,-2.0708549 3.479613,-3.2151067 c 0.130739,-0.1208454 0.302264,-0.187916 0.480351,-0.187916 v 0 c 0.629636,0 0.945662,0.7599964 0.501357,1.2058131 l -1.926779,1.9333896 -0.871247,0.7254822 z m -0.581195,1.569624 c -0.42266,0 -0.765253,-0.342365 -0.765253,-0.764686 V 2.7424664 c 0,-0.4223168 0.342593,-0.7646726 0.765253,-0.7646726 h 0.119528 c 0.42266,0 0.765298,0.3423558 0.765298,0.7646726 v 8.2576976 c 0,0.422321 -0.342638,0.764686 -0.765298,0.764686 z m 4.919799,0 c -0.230994,0 -0.449637,-0.104271 -0.594959,-0.283718 l -2.34438,-2.8950987 1.042863,-1.3190088 2.564664,3.2606405 c 0.394542,0.501685 0.03692,1.237185 -0.601702,1.237185 z", + "fill": "#3a3a3a", + "id": "path5", + "style": "stroke-width:0.0455515" + }, + "children": [] + }, + { + "type": "element", + "name": "defs", + "attributes": { + "id": "defs6" + }, + "children": [ + { + "type": "element", + "name": "linearGradient", + "attributes": { + "id": "paint0_linear_3874_31725", + "x1": "258.13101", + "y1": "269.78299", + "x2": "88.645203", + "y2": "75.4571", + "gradientUnits": "userSpaceOnUse", + "gradientTransform": "scale(0.04556973,0.04553331)" + }, + "children": [ + { + "type": "element", + "name": "stop", + "attributes": { + "stop-color": "#FB9341", + "id": "stop5" + }, + "children": [] + }, + { + "type": "element", + "name": "stop", + "attributes": { + "offset": "1", + "stop-color": "#E30D3E", + "id": "stop6" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "OpikIcon" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/tracing/OpikIcon.tsx b/web/app/components/base/icons/src/public/tracing/OpikIcon.tsx new file mode 100644 index 00000000000000..4729baae795aed --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/OpikIcon.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './OpikIcon.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'OpikIcon' + +export default Icon diff --git a/web/app/components/base/icons/src/public/tracing/OpikIconBig.json b/web/app/components/base/icons/src/public/tracing/OpikIconBig.json new file mode 100644 index 00000000000000..1372a92c0e18da --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/OpikIconBig.json @@ -0,0 +1,162 @@ +{ + "icon": { + "type": "element", + "isRootNode": true, + "name": "svg", + "attributes": { + "width": "70.700851", + "height": "24", + "viewBox": "0 0 70.700851 24", + "fill": "none", + "version": "1.1", + "id": "svg6", + "sodipodi:docname": "opik-icon-big.svg", + "inkscape:version": "1.3.2 (091e20ef0f, 2023-11-25)", + "xmlns:inkscape": "http://www.inkscape.org/namespaces/inkscape", + "xmlns:sodipodi": "http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd", + "xmlns": "http://www.w3.org/2000/svg", + "xmlns:svg": "http://www.w3.org/2000/svg" + }, + "children": [ + { + "type": "element", + "name": "sodipodi:namedview", + "attributes": { + "id": "namedview6", + "pagecolor": "#ffffff", + "bordercolor": "#666666", + "borderopacity": "1.0", + "inkscape:showpageshadow": "2", + "inkscape:pageopacity": "0.0", + "inkscape:pagecheckerboard": "0", + "inkscape:deskcolor": "#d1d1d1", + "inkscape:zoom": "18.615088", + "inkscape:cx": "36.314629", + "inkscape:cy": "18.989972", + "inkscape:window-width": "2560", + "inkscape:window-height": "1371", + "inkscape:window-x": "0", + "inkscape:window-y": "0", + "inkscape:window-maximized": "1", + "inkscape:current-layer": "svg6" + }, + "children": [] + }, + { + "type": "element", + "name": "rect", + "attributes": { + "width": "70.700851", + "height": "24", + "fill": "#ffffff", + "id": "rect1", + "x": "0", + "y": "0", + "style": "stroke-width:0.0683761;fill:none" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "fill-rule": "evenodd", + "clip-rule": "evenodd", + "d": "M 14.463316,5.8949744 C 11.191179,4.456547 7.3065299,5.9932444 5.8118769,9.3932308 4.3172308,12.793231 5.8113846,16.694496 9.0834872,18.132923 c 1.3197948,0.580171 2.7316238,0.676855 4.0492308,0.364923 0.567179,-0.13429 1.135863,0.216684 1.270085,0.783863 0.134291,0.56718 -0.216615,1.135864 -0.783863,1.270154 C 11.873983,20.964923 9.9916581,20.83788 8.2340513,20.065231 3.8540444,18.139761 1.9338872,12.969778 3.8795692,8.5437949 5.8252581,4.1177778 10.932786,2.0372103 15.312752,3.9626667 c 2.672342,1.1747624 4.432069,3.564376 4.952137,6.2470423 0.110974,0.57224 -0.262974,1.126017 -0.835214,1.236992 -0.572171,0.110906 -1.126017,-0.263043 -1.236923,-0.835282 C 17.796308,8.5668376 16.46359,6.7742906 14.463316,5.8949744 Z M 18.01388,17.557812 c 0.20424,0.856752 -0.324786,1.716786 -1.181538,1.921026 -0.856752,0.204171 -1.716855,-0.324787 -1.921026,-1.181539 -0.204239,-0.856752 0.324787,-1.716855 1.181539,-1.921025 0.856752,-0.20424 1.716854,0.324786 1.921025,1.181538 z m 1.329368,-1.216547 c 1.144615,-0.272821 1.85135,-1.42188 1.57853,-2.566427 -0.272821,-1.144616 -1.421881,-1.851351 -2.566428,-1.57853 -1.144615,0.27282 -1.85135,1.421812 -1.578529,2.566427 0.27282,1.144615 1.42188,1.85135 2.566427,1.57853 z", + "fill": "url(#paint0_linear_3874_31725)", + "id": "path1", + "style": "fill:url(#paint0_linear_3874_31725);stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 31.039658,17.805538 c -1.082803,0 -2.046769,-0.231042 -2.891897,-0.693196 -0.84506,-0.475419 -1.511932,-1.122462 -2.000479,-1.941128 -0.488615,-0.818667 -0.732855,-1.749607 -0.732855,-2.792821 0,-1.056342 0.24424,-1.987282 0.732855,-2.7928204 0.488547,-0.8186666 1.155419,-1.4590769 2.000479,-1.9212991 0.845128,-0.4621538 1.809094,-0.6931966 2.891897,-0.6931966 1.096,0 2.06653,0.2310428 2.911658,0.6931966 0.858257,0.4622222 1.525128,1.096 2.000479,1.9015385 0.488615,0.80547 0.732855,1.742974 0.732855,2.812581 0,1.043214 -0.24424,1.974154 -0.732855,2.792821 -0.475351,0.818666 -1.142222,1.465709 -2.000479,1.941128 -0.845128,0.462154 -1.815658,0.693196 -2.911658,0.693196 z m 0,-2.119316 c 0.607385,0 1.148786,-0.132102 1.624137,-0.396171 0.475419,-0.264068 0.845128,-0.647042 1.109196,-1.148786 0.277334,-0.501812 0.416,-1.089436 0.416,-1.762872 0,-0.686632 -0.138666,-1.274256 -0.416,-1.762803 -0.264068,-0.501812 -0.633777,-0.8847182 -1.109196,-1.148855 -0.475351,-0.2640683 -1.01012,-0.3961025 -1.604376,-0.3961025 -0.607385,0 -1.148787,0.1320342 -1.624137,0.3961025 -0.462222,0.2641368 -0.831932,0.647043 -1.109265,1.148855 -0.277265,0.488547 -0.415932,1.076171 -0.415932,1.762803 0,0.673436 0.138667,1.26106 0.415932,1.762872 0.277333,0.501744 0.647043,0.884718 1.109265,1.148786 0.47535,0.264069 1.01012,0.396171 1.604376,0.396171 z", + "fill": "#3a3a3a", + "id": "path2", + "style": "stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 44.915145,17.805538 c -0.858256,0 -1.643966,-0.198017 -2.35706,-0.594188 -0.699829,-0.396171 -1.261059,-0.990359 -1.683555,-1.782632 -0.409368,-0.805539 -0.614017,-1.822291 -0.614017,-3.050325 0,-1.241231 0.198017,-2.257983 0.594188,-3.0503246 0.409367,-0.7922735 0.963966,-1.3798975 1.663795,-1.7628034 0.699829,-0.396171 1.498735,-0.5941881 2.396649,-0.5941881 1.043214,0 1.960958,0.2244787 2.753231,0.6734359 0.80547,0.4489573 1.439316,1.076171 1.90147,1.881641 0.475351,0.8055382 0.713026,1.7562392 0.713026,2.8522392 0,1.096 -0.237675,2.053333 -0.713026,2.872069 -0.462154,0.80547 -1.096,1.432683 -1.90147,1.881641 -0.792273,0.448957 -1.710017,0.673435 -2.753231,0.673435 z m -4.714598,3.703932 c -0.634188,0 -1.148308,-0.514051 -1.148308,-1.148239 V 8.2381538 c 0,-0.634188 0.51412,-1.1482393 1.148308,-1.1482393 h 0.06044 c 0.634188,0 1.148308,0.5140513 1.148308,1.1482393 v 1.3474188 l -0.07925,2.8126494 0.198086,2.812581 v 5.150428 c 0,0.634188 -0.51412,1.148239 -1.148308,1.148239 z m 4.437333,-5.823248 c 0.594188,0 1.122394,-0.132102 1.584547,-0.396171 0.475351,-0.264068 0.851693,-0.647042 1.129026,-1.148786 0.277265,-0.501812 0.415932,-1.089436 0.415932,-1.762872 0,-0.686632 -0.138667,-1.274256 -0.415932,-1.762803 C 47.07412,10.113778 46.697778,9.7308718 46.222427,9.466735 45.760274,9.2026667 45.232068,9.0706325 44.63788,9.0706325 c -0.594256,0 -1.129025,0.1320342 -1.604376,0.3961025 -0.475419,0.2641368 -0.85176,0.647043 -1.129025,1.148855 -0.277334,0.488547 -0.415932,1.076171 -0.415932,1.762803 0,0.673436 0.138598,1.26106 0.415932,1.762872 0.277265,0.501744 0.653606,0.884718 1.129025,1.148786 0.475351,0.264069 1.01012,0.396171 1.604376,0.396171 z", + "fill": "#3a3a3a", + "id": "path3", + "style": "stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 53.779282,17.66694 c -0.634188,0 -1.148308,-0.514119 -1.148308,-1.148308 V 8.2381538 c 0,-0.634188 0.51412,-1.1482393 1.148308,-1.1482393 h 0.179282 c 0.634188,0 1.148308,0.5140513 1.148308,1.1482393 v 8.2804782 c 0,0.634189 -0.51412,1.148308 -1.148308,1.148308 z m 0.09956,-12.3200819 c -0.462154,0 -0.845129,-0.1452513 -1.148855,-0.4357607 -0.290462,-0.2905025 -0.435692,-0.6404307 -0.435692,-1.0497777 0,-0.4225505 0.14523,-0.7724787 0.435692,-1.0497778 0.303726,-0.2905026 0.686701,-0.4357607 1.148855,-0.4357607 0.462153,0 0.838495,0.138653 1.129025,0.4159521 0.303658,0.2640958 0.455522,0.6008205 0.455522,1.0101676 0,0.4357538 -0.145231,0.8054906 -0.435761,1.1091965 -0.290462,0.2905094 -0.673436,0.4357607 -1.148786,0.4357607 z", + "fill": "#3a3a3a", + "id": "path4", + "style": "stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "path", + "attributes": { + "d": "m 60.376821,15.30988 0.05942,-3.109743 5.22106,-4.8280344 c 0.196169,-0.1814701 0.453537,-0.2821881 0.72075,-0.2821881 v 0 c 0.944752,0 1.41894,1.141265 0.752274,1.8107351 l -2.891077,2.9033164 -1.307282,1.089436 z m -0.872069,2.35706 c -0.634188,0 -1.148239,-0.514119 -1.148239,-1.148308 V 4.1182838 c 0,-0.6341812 0.514051,-1.1482872 1.148239,-1.1482872 h 0.179351 c 0.634188,0 1.148307,0.514106 1.148307,1.1482872 V 16.518632 c 0,0.634189 -0.514119,1.148308 -1.148307,1.148308 z m 7.382017,0 c -0.346598,0 -0.674666,-0.156581 -0.892718,-0.426051 l -3.517675,-4.347487 1.564786,-1.980718 3.848206,4.89641 c 0.592,0.753368 0.05538,1.857846 -0.902838,1.857846 z", + "fill": "#3a3a3a", + "id": "path5", + "style": "stroke-width:0.0683761" + }, + "children": [] + }, + { + "type": "element", + "name": "defs", + "attributes": { + "id": "defs6" + }, + "children": [ + { + "type": "element", + "name": "linearGradient", + "attributes": { + "id": "paint0_linear_3874_31725", + "x1": "258.13101", + "y1": "269.78299", + "x2": "88.645203", + "y2": "75.4571", + "gradientUnits": "userSpaceOnUse", + "gradientTransform": "scale(0.06837607)" + }, + "children": [ + { + "type": "element", + "name": "stop", + "attributes": { + "stop-color": "#FB9341", + "id": "stop5" + }, + "children": [] + }, + { + "type": "element", + "name": "stop", + "attributes": { + "offset": "1", + "stop-color": "#E30D3E", + "id": "stop6" + }, + "children": [] + } + ] + } + ] + } + ] + }, + "name": "OpikIconBig" +} \ No newline at end of file diff --git a/web/app/components/base/icons/src/public/tracing/OpikIconBig.tsx b/web/app/components/base/icons/src/public/tracing/OpikIconBig.tsx new file mode 100644 index 00000000000000..10b41b961ce87a --- /dev/null +++ b/web/app/components/base/icons/src/public/tracing/OpikIconBig.tsx @@ -0,0 +1,16 @@ +// GENERATE BY script +// DON NOT EDIT IT MANUALLY + +import * as React from 'react' +import data from './OpikIconBig.json' +import IconBase from '@/app/components/base/icons/IconBase' +import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase' + +const Icon = React.forwardRef, Omit>(( + props, + ref, +) => ) + +Icon.displayName = 'OpikIconBig' + +export default Icon diff --git a/web/app/components/base/icons/src/public/tracing/index.ts b/web/app/components/base/icons/src/public/tracing/index.ts index 9cedf9cec39fb0..09ffd54bd4710c 100644 --- a/web/app/components/base/icons/src/public/tracing/index.ts +++ b/web/app/components/base/icons/src/public/tracing/index.ts @@ -2,4 +2,6 @@ export { default as LangfuseIconBig } from './LangfuseIconBig' export { default as LangfuseIcon } from './LangfuseIcon' export { default as LangsmithIconBig } from './LangsmithIconBig' export { default as LangsmithIcon } from './LangsmithIcon' +export { default as OpikIconBig } from './OpikIconBig' +export { default as OpikIcon } from './OpikIcon' export { default as TracingIcon } from './TracingIcon' diff --git a/web/i18n/en-US/app.ts b/web/i18n/en-US/app.ts index 861827d3e3a3f5..343b01e9b5d347 100644 --- a/web/i18n/en-US/app.ts +++ b/web/i18n/en-US/app.ts @@ -157,6 +157,10 @@ const translation = { title: 'Langfuse', description: 'Traces, evals, prompt management and metrics to debug and improve your LLM application.', }, + opik: { + title: 'Opik', + description: 'Opik is an open-source platform for evaluating, testing, and monitoring LLM applications.', + }, inUse: 'In use', configProvider: { title: 'Config ', diff --git a/web/i18n/zh-Hans/app.ts b/web/i18n/zh-Hans/app.ts index 3d3e95130d8151..be93a84195b844 100644 --- a/web/i18n/zh-Hans/app.ts +++ b/web/i18n/zh-Hans/app.ts @@ -157,6 +157,10 @@ const translation = { title: 'Langfuse', description: '跟踪、评估、提示管理和指标，以调试和改进您的 LLM 应用程序。', }, + opik: { + title: 'Opik', + description: '一个全方位的开发者平台，适用于 LLM 驱动应用程序生命周期的每个步骤。', + }, inUse: '使用中', configProvider: { title: '配置 ', diff --git a/web/models/app.ts b/web/models/app.ts index acb1c09622d270..edf8554457fae9 100644 --- a/web/models/app.ts +++ b/web/models/app.ts @@ -1,4 +1,4 @@ -import type { LangFuseConfig, LangSmithConfig, TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' +import type { LangFuseConfig, LangSmithConfig, OpikConfig, TracingProvider } from '@/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/tracing/type' import type { App, AppSSO, AppTemplate, SiteConfig } from '@/types/app' /* export type App = { @@ -165,5 +165,5 @@ export type TracingStatus = { export type TracingConfig = { tracing_provider: TracingProvider - tracing_config: LangSmithConfig | LangFuseConfig + tracing_config: LangSmithConfig | LangFuseConfig | OpikConfig } From 1859d57784b15faa68e01313f4c139da1c39c1c0 Mon Sep 17 00:00:00 2001 From: mbo Date: Mon, 13 Jan 2025 17:49:30 +0800 Subject: [PATCH 39/42] api tool support multiple env url (#12249) Co-authored-by: mabo --- api/core/tools/utils/parser.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/core/tools/utils/parser.py b/api/core/tools/utils/parser.py index 30e4fdcf06dbc4..9d88d6d6ef5ce3 100644 --- a/api/core/tools/utils/parser.py +++ b/api/core/tools/utils/parser.py @@ -5,6 +5,7 @@ from json.decoder import JSONDecodeError from typing import Optional +from flask import request from requests import get from yaml import YAMLError, safe_load # type: ignore @@ -29,6 +30,10 @@ def parse_openapi_to_tool_bundle( raise ToolProviderNotFoundError("No server found in the openapi yaml.") server_url = openapi["servers"][0]["url"] + request_env = request.headers.get("X-Request-Env") + if request_env: + matched_servers = [server["url"] for server in openapi["servers"] if server["env"] == request_env] + server_url = matched_servers[0] if matched_servers else server_url # list all interfaces interfaces = [] From b4873ecb435c79ac1385157efac8843d33781da5 Mon Sep 17 00:00:00 2001 From: Warren Chen Date: Mon, 13 Jan 2025 18:29:06 +0800 Subject: [PATCH 40/42] [fix] support feature restore (#12563) --- .../workflow/hooks/use-workflow-run.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/web/app/components/workflow/hooks/use-workflow-run.ts b/web/app/components/workflow/hooks/use-workflow-run.ts index 11f0a1973f9ba9..53a6b5846552f4 100644 --- a/web/app/components/workflow/hooks/use-workflow-run.ts +++ b/web/app/components/workflow/hooks/use-workflow-run.ts @@ -760,7 +760,21 @@ export const useWorkflowRun = () => { edges, viewport, }) - featuresStore?.setState({ features: publishedWorkflow.features }) + const mappedFeatures = { + opening: { + enabled: !!publishedWorkflow.features.opening_statement || !!publishedWorkflow.features.suggested_questions.length, + opening_statement: publishedWorkflow.features.opening_statement, + suggested_questions: publishedWorkflow.features.suggested_questions, + }, + suggested: publishedWorkflow.features.suggested_questions_after_answer, + text2speech: publishedWorkflow.features.text_to_speech, + speech2text: publishedWorkflow.features.speech_to_text, + citation: publishedWorkflow.features.retriever_resource, + moderation: publishedWorkflow.features.sensitive_word_avoidance, + file: publishedWorkflow.features.file_upload, + } + + featuresStore?.setState({ features: mappedFeatures }) workflowStore.getState().setPublishedAt(publishedWorkflow.created_at) workflowStore.getState().setEnvironmentVariables(publishedWorkflow.environment_variables || []) }, [featuresStore, handleUpdateWorkflowCanvas, workflowStore]) From 1e9ac7ffebec3827d4d9c459c5de5636e6f71ea5 Mon Sep 17 00:00:00 2001 From: eux Date: Mon, 13 Jan 2025 18:31:43 +0800 Subject: [PATCH 41/42] feat: add table of contents to Knowledge API doc (#12688) --- web/app/(commonLayout)/datasets/Container.tsx | 2 +- web/app/(commonLayout)/datasets/Doc.tsx | 107 ++++++++++++++++-- 2 files changed, 96 insertions(+), 13 deletions(-) diff --git a/web/app/(commonLayout)/datasets/Container.tsx b/web/app/(commonLayout)/datasets/Container.tsx index c39d9c5dbfd69b..f484d30a3d3f8d 100644 --- a/web/app/(commonLayout)/datasets/Container.tsx +++ b/web/app/(commonLayout)/datasets/Container.tsx @@ -82,7 +82,7 @@ const Container = () => { }, [currentWorkspace, router]) return ( -
    +
    = ({ - apiBaseUrl, -}) => { + +const Doc = ({ apiBaseUrl }: DocProps) => { const { locale } = useContext(I18n) + const { t } = useTranslation() + const [toc, setToc] = useState>([]) + const [isTocExpanded, setIsTocExpanded] = useState(false) + // Set initial TOC expanded state based on screen width useEffect(() => { - const hash = location.hash - if (hash) - document.querySelector(hash)?.scrollIntoView() + const mediaQuery = window.matchMedia('(min-width: 1280px)') + setIsTocExpanded(mediaQuery.matches) }, []) + // Extract TOC from article content + useEffect(() => { + const extractTOC = () => { + const article = document.querySelector('article') + if (article) { + const headings = article.querySelectorAll('h2') + const tocItems = Array.from(headings).map((heading) => { + const anchor = heading.querySelector('a') + if (anchor) { + return { + href: anchor.getAttribute('href') || '', + text: anchor.textContent || '', + } + } + return null + }).filter((item): item is { href: string; text: string } => item !== null) + setToc(tocItems) + } + } + + setTimeout(extractTOC, 0) + }, [locale]) + + // Handle TOC item click + const handleTocClick = (e: React.MouseEvent, item: { href: string; text: string }) => { + e.preventDefault() + const targetId = item.href.replace('#', '') + const element = document.getElementById(targetId) + if (element) { + const scrollContainer = document.querySelector('.scroll-container') + if (scrollContainer) { + const headerOffset = -40 + const elementTop = element.offsetTop - headerOffset + scrollContainer.scrollTo({ + top: elementTop, + behavior: 'smooth', + }) + } + } + } + return ( -
    - { - locale !== LanguagesSupported[1] +
    +
    + {isTocExpanded + ? ( + + ) + : ( + + )} +
    +
    + {locale !== LanguagesSupported[1] ? : - } -
    + } +
    +
    ) } From 6e0fb055d18969eb923e719ad92ecac3a5c5d534 Mon Sep 17 00:00:00 2001 From: -LAN- Date: Mon, 13 Jan 2025 19:21:06 +0800 Subject: [PATCH 42/42] chore: bump version to 0.15.1 (#12690) Signed-off-by: -LAN- --- api/configs/packaging/__init__.py | 2 +- docker-legacy/docker-compose.yaml | 6 +++--- docker/docker-compose-template.yaml | 6 +++--- docker/docker-compose.yaml | 6 +++--- web/package.json | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/configs/packaging/__init__.py b/api/configs/packaging/__init__.py index 278b1d3b8f4a83..a54c5bf5ee6c7b 100644 --- a/api/configs/packaging/__init__.py +++ b/api/configs/packaging/__init__.py @@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings): CURRENT_VERSION: str = Field( description="Dify version", - default="0.15.0", + default="0.15.1", ) COMMIT_SHA: str = Field( diff --git a/docker-legacy/docker-compose.yaml b/docker-legacy/docker-compose.yaml index c8bf382bcd4f0f..6e4c8a748ea48b 100644 --- a/docker-legacy/docker-compose.yaml +++ b/docker-legacy/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3' services: # API service api: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Startup mode, 'api' starts the API server. @@ -227,7 +227,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: CONSOLE_WEB_URL: '' @@ -397,7 +397,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.0 + image: langgenius/dify-web:0.15.1 restart: always environment: # The base URL of console application api server, refers to the Console base URL of WEB service if console domain is diff --git a/docker/docker-compose-template.yaml b/docker/docker-compose-template.yaml index 6d70f14424ff1a..e2daead92e1f15 100644 --- a/docker/docker-compose-template.yaml +++ b/docker/docker-compose-template.yaml @@ -2,7 +2,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Use the shared environment variables. @@ -25,7 +25,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Use the shared environment variables. @@ -47,7 +47,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.0 + image: langgenius/dify-web:0.15.1 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 173a88bc4cd933..f60fcdbcfcc619 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -393,7 +393,7 @@ x-shared-env: &shared-api-worker-env services: # API service api: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Use the shared environment variables. @@ -416,7 +416,7 @@ services: # worker service # The Celery worker for processing the queue. worker: - image: langgenius/dify-api:0.15.0 + image: langgenius/dify-api:0.15.1 restart: always environment: # Use the shared environment variables. @@ -438,7 +438,7 @@ services: # Frontend web application. web: - image: langgenius/dify-web:0.15.0 + image: langgenius/dify-web:0.15.1 restart: always environment: CONSOLE_API_URL: ${CONSOLE_API_URL:-} diff --git a/web/package.json b/web/package.json index 7afb766d87b21d..879b87c596a60e 100644 --- a/web/package.json +++ b/web/package.json @@ -1,6 +1,6 @@ { "name": "dify-web", - "version": "0.15.0", + "version": "0.15.1", "private": true, "engines": { "node": ">=18.17.0"