chore: optimize asynchronous workflow deletion performance of app related data

Co-authored-by: dufei <[email protected]>

.github/workflows/api-tests.yml

@@ -89,6 +89,5 @@ jobs:
             pgvecto-rs
             pgvector
             chroma
-            myscale
       - name: Test Vector Stores
         run: poetry run -C api bash dev/pytest/pytest_vdb.sh

api/core/app/app_config/easy_ui_based_app/dataset/manager.py

@@ -62,7 +62,12 @@ class DatasetConfigManager:
             return None
 
         # dataset configs
-        dataset_configs = config.get('dataset_configs', {'retrieval_model': 'single'})
+        if 'dataset_configs' in config and config.get('dataset_configs'):
+            dataset_configs = config.get('dataset_configs')
+        else:
+            dataset_configs = {
+                'retrieval_model': 'multiple'
+            }
         query_variable = config.get('dataset_query_variable')
 
         if dataset_configs['retrieval_model'] == 'single':
@@ -83,9 +88,10 @@ class DatasetConfigManager:
                     retrieve_strategy=DatasetRetrieveConfigEntity.RetrieveStrategy.value_of(
                         dataset_configs['retrieval_model']
                     ),
-                    top_k=dataset_configs.get('top_k'),
+                    top_k=dataset_configs.get('top_k', 4),
                     score_threshold=dataset_configs.get('score_threshold'),
-                    reranking_model=dataset_configs.get('reranking_model')
+                    reranking_model=dataset_configs.get('reranking_model'),
+                    weights=dataset_configs.get('weights')
                 )
             )
 

api/core/app/app_config/entities.py

@@ -159,7 +159,11 @@ class DatasetRetrieveConfigEntity(BaseModel):
     retrieve_strategy: RetrieveStrategy
     top_k: Optional[int] = None
     score_threshold: Optional[float] = None
+    rerank_mode: Optional[str] = 'reranking_model'
     reranking_model: Optional[dict] = None
+    weights: Optional[dict] = None
+
+
 
 
 class DatasetEntity(BaseModel):

api/core/model_runtime/model_providers/groq/llm/_position.yaml

@@ -0,0 +1,7 @@
+- llama-3.1-405b-reasoning
+- llama-3.1-70b-versatile
+- llama-3.1-8b-instant
+- llama3-70b-8192
+- llama3-8b-8192
+- mixtral-8x7b-32768
+- llama2-70b-4096

api/core/model_runtime/model_providers/groq/llm/llama-3.1-405b-reasoning.yaml

@@ -0,0 +1,25 @@
+model: llama-3.1-405b-reasoning
+label:
+  zh_Hans: Llama-3.1-405b-reasoning
+  en_US: Llama-3.1-405b-reasoning
+model_type: llm
+features:
+  - agent-thought
+model_properties:
+  mode: chat
+  context_size: 131072
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 512
+    min: 1
+    max: 8192
+pricing:
+  input: '0.05'
+  output: '0.1'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/groq/llm/llama-3.1-70b-versatile.yaml

@@ -0,0 +1,25 @@
+model: llama-3.1-70b-versatile
+label:
+  zh_Hans: Llama-3.1-70b-versatile
+  en_US: Llama-3.1-70b-versatile
+model_type: llm
+features:
+  - agent-thought
+model_properties:
+  mode: chat
+  context_size: 131072
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 512
+    min: 1
+    max: 8192
+pricing:
+  input: '0.05'
+  output: '0.1'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/groq/llm/llama-3.1-8b-instant.yaml

@@ -0,0 +1,25 @@
+model: llama-3.1-8b-instant
+label:
+  zh_Hans: Llama-3.1-8b-instant
+  en_US: Llama-3.1-8b-instant
+model_type: llm
+features:
+  - agent-thought
+model_properties:
+  mode: chat
+  context_size: 131072
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    default: 512
+    min: 1
+    max: 8192
+pricing:
+  input: '0.05'
+  output: '0.1'
+  unit: '0.000001'
+  currency: USD

api/core/model_runtime/model_providers/openrouter/llm/_position.yaml

@@ -12,6 +12,9 @@
 - google/gemini-pro
 - cohere/command-r-plus
 - cohere/command-r
+- meta-llama/llama-3.1-405b-instruct
+- meta-llama/llama-3.1-70b-instruct
+- meta-llama/llama-3.1-8b-instruct
 - meta-llama/llama-3-70b-instruct
 - meta-llama/llama-3-8b-instruct
 - mistralai/mixtral-8x22b-instruct

api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-405b-instruct.yaml

@@ -0,0 +1,23 @@
+model: meta-llama/llama-3.1-405b-instruct
+label:
+  en_US: llama-3.1-405b-instruct
+model_type: llm
+model_properties:
+  mode: chat
+  context_size: 128000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    required: true
+    default: 512
+    min: 1
+    max: 128000
+pricing:
+  input: "3"
+  output: "3"
+  unit: "0.000001"
+  currency: USD

api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-70b-instruct.yaml

@@ -0,0 +1,23 @@
+model: meta-llama/llama-3.1-70b-instruct
+label:
+  en_US: llama-3.1-70b-instruct
+model_type: llm
+model_properties:
+  mode: chat
+  context_size: 128000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    required: true
+    default: 512
+    min: 1
+    max: 128000
+pricing:
+  input: "0.9"
+  output: "0.9"
+  unit: "0.000001"
+  currency: USD

api/core/model_runtime/model_providers/openrouter/llm/llama-3.1-8b-instruct.yaml

@@ -0,0 +1,23 @@
+model: meta-llama/llama-3.1-8b-instruct
+label:
+  en_US: llama-3.1-8b-instruct
+model_type: llm
+model_properties:
+  mode: chat
+  context_size: 128000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+  - name: top_p
+    use_template: top_p
+  - name: max_tokens
+    use_template: max_tokens
+    required: true
+    default: 512
+    min: 1
+    max: 128000
+pricing:
+  input: "0.2"
+  output: "0.2"
+  unit: "0.000001"
+  currency: USD

api/core/model_runtime/model_providers/tongyi/llm/llm.py

@@ -425,7 +425,7 @@ You should also complete the text started with ``` but not tell ``` directly.
             raise ValueError(f"Got unknown type {message}")
 
         return message_text
-    
+
     def _convert_messages_to_prompt(self, messages: list[PromptMessage]) -> str:
         """
         Format a list of messages into a full prompt for the Anthropic model
@@ -500,6 +500,9 @@ You should also complete the text started with ``` but not tell ``` directly.
                 tongyi_messages.append({
                     'role': 'assistant',
                     'content': content if not rich_content else [{"text": content}],
+                    'tool_calls': [tool_call.model_dump() for tool_call in
+                                   prompt_message.tool_calls] if prompt_message.tool_calls else []
+
                 })
             elif isinstance(prompt_message, ToolPromptMessage):
                 tongyi_messages.append({

api/core/ops/utils.py

@@ -25,7 +25,6 @@ def measure_time():
         yield timing_info
     finally:
         timing_info['end'] = datetime.now()
-        print(f"Execution time: {timing_info['end'] - timing_info['start']}")
 
 
 def replace_text_with_content(data):

api/core/rag/data_post_processor/data_post_processor.py

@@ -5,15 +5,20 @@ from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.invoke import InvokeAuthorizationError
 from core.rag.data_post_processor.reorder import ReorderRunner
 from core.rag.models.document import Document
-from core.rag.rerank.rerank import RerankRunner
+from core.rag.rerank.constants.rerank_mode import RerankMode
+from core.rag.rerank.entity.weight import KeywordSetting, VectorSetting, Weights
+from core.rag.rerank.rerank_model import RerankModelRunner
+from core.rag.rerank.weight_rerank import WeightRerankRunner
 
 
 class DataPostProcessor:
     """Interface for data post-processing document.
     """
 
-    def __init__(self, tenant_id: str, reranking_model: dict, reorder_enabled: bool = False):
-        self.rerank_runner = self._get_rerank_runner(reranking_model, tenant_id)
+    def __init__(self, tenant_id: str, reranking_mode: str,
+                 reranking_model: Optional[dict] = None, weights: Optional[dict] = None,
+                 reorder_enabled: bool = False):
+        self.rerank_runner = self._get_rerank_runner(reranking_mode, tenant_id, reranking_model, weights)
         self.reorder_runner = self._get_reorder_runner(reorder_enabled)
 
     def invoke(self, query: str, documents: list[Document], score_threshold: Optional[float] = None,
@@ -26,19 +31,37 @@ class DataPostProcessor:
 
         return documents
 
-    def _get_rerank_runner(self, reranking_model: dict, tenant_id: str) -> Optional[RerankRunner]:
-        if reranking_model:
-            try:
-                model_manager = ModelManager()
-                rerank_model_instance = model_manager.get_model_instance(
-                    tenant_id=tenant_id,
-                    provider=reranking_model['reranking_provider_name'],
-                    model_type=ModelType.RERANK,
-                    model=reranking_model['reranking_model_name']
+    def _get_rerank_runner(self, reranking_mode: str, tenant_id: str, reranking_model: Optional[dict] = None,
+                           weights: Optional[dict] = None) -> Optional[RerankModelRunner | WeightRerankRunner]:
+        if reranking_mode == RerankMode.WEIGHTED_SCORE.value and weights:
+            return WeightRerankRunner(
+                tenant_id,
+                Weights(
+                    weight_type=weights['weight_type'],
+                    vector_setting=VectorSetting(
+                        vector_weight=weights['vector_setting']['vector_weight'],
+                        embedding_provider_name=weights['vector_setting']['embedding_provider_name'],
+                        embedding_model_name=weights['vector_setting']['embedding_model_name'],
+                    ),
+                    keyword_setting=KeywordSetting(
+                        keyword_weight=weights['keyword_setting']['keyword_weight'],
+                    )
                 )
-            except InvokeAuthorizationError:
-                return None
-            return RerankRunner(rerank_model_instance)
+            )
+        elif reranking_mode == RerankMode.RERANKING_MODEL.value:
+            if reranking_model:
+                try:
+                    model_manager = ModelManager()
+                    rerank_model_instance = model_manager.get_model_instance(
+                        tenant_id=tenant_id,
+                        provider=reranking_model['reranking_provider_name'],
+                        model_type=ModelType.RERANK,
+                        model=reranking_model['reranking_model_name']
+                    )
+                except InvokeAuthorizationError:
+                    return None
+                return RerankModelRunner(rerank_model_instance)
+            return None
         return None
 
     def _get_reorder_runner(self, reorder_enabled) -> Optional[ReorderRunner]:

api/core/rag/datasource/keyword/jieba/jieba_keyword_table_handler.py

@@ -1,4 +1,5 @@
 import re
+from typing import Optional
 
 import jieba
 from jieba.analyse import default_tfidf
@@ -11,7 +12,7 @@ class JiebaKeywordTableHandler:
     def __init__(self):
         default_tfidf.stop_words = STOPWORDS
 
-    def extract_keywords(self, text: str, max_keywords_per_chunk: int = 10) -> set[str]:
+    def extract_keywords(self, text: str, max_keywords_per_chunk: Optional[int] = 10) -> set[str]:
         """Extract keywords with JIEBA tfidf."""
         keywords = jieba.analyse.extract_tags(
             sentence=text,

api/core/rag/datasource/retrieval_service.py

@@ -6,6 +6,7 @@ from flask import Flask, current_app
 from core.rag.data_post_processor.data_post_processor import DataPostProcessor
 from core.rag.datasource.keyword.keyword_factory import Keyword
 from core.rag.datasource.vdb.vector_factory import Vector
+from core.rag.rerank.constants.rerank_mode import RerankMode
 from core.rag.retrieval.retrival_methods import RetrievalMethod
 from extensions.ext_database import db
 from models.dataset import Dataset
@@ -26,13 +27,19 @@ class RetrievalService:
 
     @classmethod
     def retrieve(cls, retrival_method: str, dataset_id: str, query: str,
-                 top_k: int, score_threshold: Optional[float] = .0, reranking_model: Optional[dict] = None):
+                 top_k: int, score_threshold: Optional[float] = .0,
+                 reranking_model: Optional[dict] = None, reranking_mode: Optional[str] = None,
+                 weights: Optional[dict] = None):
         dataset = db.session.query(Dataset).filter(
             Dataset.id == dataset_id
         ).first()
         if not dataset or dataset.available_document_count == 0 or dataset.available_segment_count == 0:
             return []
         all_documents = []
+        keyword_search_documents = []
+        embedding_search_documents = []
+        full_text_search_documents = []
+        hybrid_search_documents = []
         threads = []
         exceptions = []
         # retrieval_model source with keyword
@@ -87,7 +94,8 @@ class RetrievalService:
             raise Exception(exception_message)
 
         if retrival_method == RetrievalMethod.HYBRID_SEARCH.value:
-            data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False)
+            data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_mode,
+                                                    reranking_model, weights, False)
             all_documents = data_post_processor.invoke(
                 query=query,
                 documents=all_documents,
@@ -143,7 +151,9 @@ class RetrievalService:
 
                 if documents:
                     if reranking_model and retrival_method == RetrievalMethod.SEMANTIC_SEARCH.value:
-                        data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False)
+                        data_post_processor = DataPostProcessor(str(dataset.tenant_id),
+                                                                RerankMode.RERANKING_MODEL.value,
+                                                                reranking_model, None, False)
                         all_documents.extend(data_post_processor.invoke(
                             query=query,
                             documents=documents,
@@ -175,7 +185,9 @@ class RetrievalService:
                 )
                 if documents:
                     if reranking_model and retrival_method == RetrievalMethod.FULL_TEXT_SEARCH.value:
-                        data_post_processor = DataPostProcessor(str(dataset.tenant_id), reranking_model, False)
+                        data_post_processor = DataPostProcessor(str(dataset.tenant_id),
+                                                                RerankMode.RERANKING_MODEL.value,
+                                                                reranking_model, None, False)
                         all_documents.extend(data_post_processor.invoke(
                             query=query,
                             documents=documents,

api/core/rag/datasource/vdb/qdrant/qdrant_vector.py

@@ -396,9 +396,11 @@ class QdrantVector(BaseVector):
         documents = []
         for result in results:
             if result:
-                documents.append(self._document_from_scored_point(
+                document = self._document_from_scored_point(
                     result, Field.CONTENT_KEY.value, Field.METADATA_KEY.value
-                ))
+                )
+                document.metadata['vector'] = result.vector
+                documents.append(document)
 
         return documents
 

api/core/rag/rerank/constants/rerank_mode.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class RerankMode(Enum):
+
+    RERANKING_MODEL = 'reranking_model'
+    WEIGHTED_SCORE = 'weighted_score'
+

api/core/rag/rerank/entity/weight.py

@@ -0,0 +1,23 @@
+from pydantic import BaseModel
+
+
+class VectorSetting(BaseModel):
+    vector_weight: float
+
+    embedding_provider_name: str
+
+    embedding_model_name: str
+
+
+class KeywordSetting(BaseModel):
+    keyword_weight: float
+
+
+class Weights(BaseModel):
+    """Model for weighted rerank."""
+
+    weight_type: str
+
+    vector_setting: VectorSetting
+
+    keyword_setting: KeywordSetting

api/core/rag/rerank/rerank_model.py

@@ -4,7 +4,7 @@ from core.model_manager import ModelInstance
 from core.rag.models.document import Document
 
 
-class RerankRunner:
+class RerankModelRunner:
     def __init__(self, rerank_model_instance: ModelInstance) -> None:
         self.rerank_model_instance = rerank_model_instance
 

api/core/rag/rerank/weight_rerank.py

@@ -0,0 +1,178 @@
+import math
+from collections import Counter
+from typing import Optional
+
+import numpy as np
+
+from core.embedding.cached_embedding import CacheEmbedding
+from core.model_manager import ModelManager
+from core.model_runtime.entities.model_entities import ModelType
+from core.rag.datasource.keyword.jieba.jieba_keyword_table_handler import JiebaKeywordTableHandler
+from core.rag.models.document import Document
+from core.rag.rerank.entity.weight import VectorSetting, Weights
+
+
+class WeightRerankRunner:
+
+    def __init__(self, tenant_id: str, weights: Weights) -> None:
+        self.tenant_id = tenant_id
+        self.weights = weights
+
+    def run(self, query: str, documents: list[Document], score_threshold: Optional[float] = None,
+            top_n: Optional[int] = None, user: Optional[str] = None) -> list[Document]:
+        """
+        Run rerank model
+        :param query: search query
+        :param documents: documents for reranking
+        :param score_threshold: score threshold
+        :param top_n: top n
+        :param user: unique user id if needed
+
+        :return:
+        """
+        docs = []
+        doc_id = []
+        unique_documents = []
+        for document in documents:
+            if document.metadata['doc_id'] not in doc_id:
+                doc_id.append(document.metadata['doc_id'])
+                docs.append(document.page_content)
+                unique_documents.append(document)
+
+        documents = unique_documents
+
+        rerank_documents = []
+        query_scores = self._calculate_keyword_score(query, documents)
+
+        query_vector_scores = self._calculate_cosine(self.tenant_id, query, documents, self.weights.vector_setting)
+        for document, query_score, query_vector_score in zip(documents, query_scores, query_vector_scores):
+            # format document
+            score = self.weights.vector_setting.vector_weight * query_vector_score + \
+                    self.weights.keyword_setting.keyword_weight * query_score
+            if score_threshold and score < score_threshold:
+                continue
+            document.metadata['score'] = score
+            rerank_documents.append(document)
+        rerank_documents = sorted(rerank_documents, key=lambda x: x.metadata['score'], reverse=True)
+        return rerank_documents[:top_n] if top_n else rerank_documents
+
+    def _calculate_keyword_score(self, query: str, documents: list[Document]) -> list[float]:
+        """
+        Calculate BM25 scores
+        :param query: search query
+        :param documents: documents for reranking
+
+        :return:
+        """
+        keyword_table_handler = JiebaKeywordTableHandler()
+        query_keywords = keyword_table_handler.extract_keywords(query, None)
+        documents_keywords = []
+        for document in documents:
+            # get the document keywords
+            document_keywords = keyword_table_handler.extract_keywords(document.page_content, None)
+            document.metadata['keywords'] = document_keywords
+            documents_keywords.append(document_keywords)
+
+        # Counter query keywords(TF)
+        query_keyword_counts = Counter(query_keywords)
+
+        # total documents
+        total_documents = len(documents)
+
+        # calculate all documents' keywords IDF
+        all_keywords = set()
+        for document_keywords in documents_keywords:
+            all_keywords.update(document_keywords)
+
+        keyword_idf = {}
+        for keyword in all_keywords:
+            # calculate include query keywords' documents
+            doc_count_containing_keyword = sum(1 for doc_keywords in documents_keywords if keyword in doc_keywords)
+            # IDF
+            keyword_idf[keyword] = math.log((1 + total_documents) / (1 + doc_count_containing_keyword)) + 1
+
+        query_tfidf = {}
+
+        for keyword, count in query_keyword_counts.items():
+            tf = count
+            idf = keyword_idf.get(keyword, 0)
+            query_tfidf[keyword] = tf * idf
+
+        # calculate all documents' TF-IDF
+        documents_tfidf = []
+        for document_keywords in documents_keywords:
+            document_keyword_counts = Counter(document_keywords)
+            document_tfidf = {}
+            for keyword, count in document_keyword_counts.items():
+                tf = count
+                idf = keyword_idf.get(keyword, 0)
+                document_tfidf[keyword] = tf * idf
+            documents_tfidf.append(document_tfidf)
+
+        def cosine_similarity(vec1, vec2):
+            intersection = set(vec1.keys()) & set(vec2.keys())
+            numerator = sum(vec1[x] * vec2[x] for x in intersection)
+
+            sum1 = sum(vec1[x] ** 2 for x in vec1.keys())
+            sum2 = sum(vec2[x] ** 2 for x in vec2.keys())
+            denominator = math.sqrt(sum1) * math.sqrt(sum2)
+
+            if not denominator:
+                return 0.0
+            else:
+                return float(numerator) / denominator
+
+        similarities = []
+        for document_tfidf in documents_tfidf:
+            similarity = cosine_similarity(query_tfidf, document_tfidf)
+            similarities.append(similarity)
+
+        # for idx, similarity in enumerate(similarities):
+        #     print(f"Document {idx + 1} similarity: {similarity}")
+
+        return similarities
+
+    def _calculate_cosine(self, tenant_id: str, query: str, documents: list[Document],
+                          vector_setting: VectorSetting) -> list[float]:
+        """
+        Calculate Cosine scores
+        :param query: search query
+        :param documents: documents for reranking
+
+        :return:
+        """
+        query_vector_scores = []
+
+        model_manager = ModelManager()
+
+        embedding_model = model_manager.get_model_instance(
+            tenant_id=tenant_id,
+            provider=vector_setting.embedding_provider_name,
+            model_type=ModelType.TEXT_EMBEDDING,
+            model=vector_setting.embedding_model_name
+
+        )
+        cache_embedding = CacheEmbedding(embedding_model)
+        query_vector = cache_embedding.embed_query(query)
+        for document in documents:
+            # calculate cosine similarity
+            if 'score' in document.metadata:
+                query_vector_scores.append(document.metadata['score'])
+            else:
+                content_vector = document.metadata['vector']
+                # transform to NumPy
+                vec1 = np.array(query_vector)
+                vec2 = np.array(document.metadata['vector'])
+
+                # calculate dot product
+                dot_product = np.dot(vec1, vec2)
+
+                # calculate norm
+                norm_vec1 = np.linalg.norm(vec1)
+                norm_vec2 = np.linalg.norm(vec2)
+
+                # calculate cosine similarity
+                cosine_sim = dot_product / (norm_vec1 * norm_vec2)
+                query_vector_scores.append(cosine_sim)
+
+        return query_vector_scores

api/core/rag/retrieval/dataset_retrieval.py

@@ -1,4 +1,6 @@
+import math
 import threading
+from collections import Counter
 from typing import Optional, cast
 
 from flask import Flask, current_app
@@ -14,9 +16,10 @@ from core.model_runtime.entities.model_entities import ModelFeature, ModelType
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.ops.ops_trace_manager import TraceQueueManager, TraceTask, TraceTaskName
 from core.ops.utils import measure_time
+from core.rag.data_post_processor.data_post_processor import DataPostProcessor
+from core.rag.datasource.keyword.jieba.jieba_keyword_table_handler import JiebaKeywordTableHandler
 from core.rag.datasource.retrieval_service import RetrievalService
 from core.rag.models.document import Document
-from core.rag.rerank.rerank import RerankRunner
 from core.rag.retrieval.retrival_methods import RetrievalMethod
 from core.rag.retrieval.router.multi_dataset_function_call_router import FunctionCallMultiDatasetRouter
 from core.rag.retrieval.router.multi_dataset_react_route import ReactMultiDatasetRouter
@@ -132,8 +135,9 @@ class DatasetRetrieval:
                 app_id, tenant_id, user_id, user_from,
                 available_datasets, query, retrieve_config.top_k,
                 retrieve_config.score_threshold,
-                retrieve_config.reranking_model.get('reranking_provider_name'),
-                retrieve_config.reranking_model.get('reranking_model_name'),
+                retrieve_config.rerank_mode,
+                retrieve_config.reranking_model,
+                retrieve_config.weights,
                 message_id,
             )
 
@@ -272,7 +276,8 @@ class DatasetRetrieval:
                         retrival_method=retrival_method, dataset_id=dataset.id,
                         query=query,
                         top_k=top_k, score_threshold=score_threshold,
-                        reranking_model=reranking_model
+                        reranking_model=reranking_model,
+                        weights=retrieval_model_config.get('weights', None),
                     )
                 self._on_query(query, [dataset_id], app_id, user_from, user_id)
 
@@ -292,14 +297,18 @@ class DatasetRetrieval:
             query: str,
             top_k: int,
             score_threshold: float,
-            reranking_provider_name: str,
-            reranking_model_name: str,
+            reranking_mode: str,
+            reranking_model: Optional[dict] = None,
+            weights: Optional[dict] = None,
+            reranking_enable: bool = True,
             message_id: Optional[str] = None,
     ):
         threads = []
         all_documents = []
         dataset_ids = [dataset.id for dataset in available_datasets]
+        index_type = None
         for dataset in available_datasets:
+            index_type = dataset.indexing_technique
             retrieval_thread = threading.Thread(target=self._retriever, kwargs={
                 'flask_app': current_app._get_current_object(),
                 'dataset_id': dataset.id,
@@ -311,23 +320,24 @@ class DatasetRetrieval:
             retrieval_thread.start()
         for thread in threads:
             thread.join()
-        # do rerank for searched documents
-        model_manager = ModelManager()
-        rerank_model_instance = model_manager.get_model_instance(
-            tenant_id=tenant_id,
-            provider=reranking_provider_name,
-            model_type=ModelType.RERANK,
-            model=reranking_model_name
-        )
 
-        rerank_runner = RerankRunner(rerank_model_instance)
+        if reranking_enable:
+            # do rerank for searched documents
+            data_post_processor = DataPostProcessor(tenant_id, reranking_mode,
+                                                    reranking_model, weights, False)
 
-        with measure_time() as timer:
-            all_documents = rerank_runner.run(
-                query, all_documents,
-                score_threshold,
-                top_k
-            )
+            with measure_time() as timer:
+                all_documents = data_post_processor.invoke(
+                    query=query,
+                    documents=all_documents,
+                    score_threshold=score_threshold,
+                    top_n=top_k
+                )
+        else:
+            if index_type == "economy":
+                all_documents = self.calculate_keyword_score(query, all_documents, top_k)
+            elif index_type == "high_quality":
+                all_documents = self.calculate_vector_score(all_documents, top_k, score_threshold)
         self._on_query(query, dataset_ids, app_id, user_from, user_id)
 
         if all_documents:
@@ -420,7 +430,8 @@ class DatasetRetrieval:
                                                           score_threshold=retrieval_model['score_threshold']
                                                           if retrieval_model['score_threshold_enabled'] else None,
                                                           reranking_model=retrieval_model['reranking_model']
-                                                          if retrieval_model['reranking_enable'] else None
+                                                          if retrieval_model['reranking_enable'] else None,
+                                                          weights=retrieval_model.get('weights', None),
                                                           )
 
                     all_documents.extend(documents)
@@ -513,3 +524,94 @@ class DatasetRetrieval:
             tools.append(tool)
 
         return tools
+
+    def calculate_keyword_score(self, query: str, documents: list[Document], top_k: int) -> list[Document]:
+        """
+        Calculate keywords scores
+        :param query: search query
+        :param documents: documents for reranking
+
+        :return:
+        """
+        keyword_table_handler = JiebaKeywordTableHandler()
+        query_keywords = keyword_table_handler.extract_keywords(query, None)
+        documents_keywords = []
+        for document in documents:
+            # get the document keywords
+            document_keywords = keyword_table_handler.extract_keywords(document.page_content, None)
+            document.metadata['keywords'] = document_keywords
+            documents_keywords.append(document_keywords)
+
+        # Counter query keywords(TF)
+        query_keyword_counts = Counter(query_keywords)
+
+        # total documents
+        total_documents = len(documents)
+
+        # calculate all documents' keywords IDF
+        all_keywords = set()
+        for document_keywords in documents_keywords:
+            all_keywords.update(document_keywords)
+
+        keyword_idf = {}
+        for keyword in all_keywords:
+            # calculate include query keywords' documents
+            doc_count_containing_keyword = sum(1 for doc_keywords in documents_keywords if keyword in doc_keywords)
+            # IDF
+            keyword_idf[keyword] = math.log((1 + total_documents) / (1 + doc_count_containing_keyword)) + 1
+
+        query_tfidf = {}
+
+        for keyword, count in query_keyword_counts.items():
+            tf = count
+            idf = keyword_idf.get(keyword, 0)
+            query_tfidf[keyword] = tf * idf
+
+        # calculate all documents' TF-IDF
+        documents_tfidf = []
+        for document_keywords in documents_keywords:
+            document_keyword_counts = Counter(document_keywords)
+            document_tfidf = {}
+            for keyword, count in document_keyword_counts.items():
+                tf = count
+                idf = keyword_idf.get(keyword, 0)
+                document_tfidf[keyword] = tf * idf
+            documents_tfidf.append(document_tfidf)
+
+        def cosine_similarity(vec1, vec2):
+            intersection = set(vec1.keys()) & set(vec2.keys())
+            numerator = sum(vec1[x] * vec2[x] for x in intersection)
+
+            sum1 = sum(vec1[x] ** 2 for x in vec1.keys())
+            sum2 = sum(vec2[x] ** 2 for x in vec2.keys())
+            denominator = math.sqrt(sum1) * math.sqrt(sum2)
+
+            if not denominator:
+                return 0.0
+            else:
+                return float(numerator) / denominator
+
+        similarities = []
+        for document_tfidf in documents_tfidf:
+            similarity = cosine_similarity(query_tfidf, document_tfidf)
+            similarities.append(similarity)
+
+        for document, score in zip(documents, similarities):
+            # format document
+            document.metadata['score'] = score
+        documents = sorted(documents, key=lambda x: x.metadata['score'], reverse=True)
+        return documents[:top_k] if top_k else documents
+
+    def calculate_vector_score(self, all_documents: list[Document],
+                               top_k: int, score_threshold: float) -> list[Document]:
+        filter_documents = []
+        for document in all_documents:
+            if document.metadata['score'] >= score_threshold:
+                filter_documents.append(document)
+        if not filter_documents:
+            return []
+        filter_documents = sorted(filter_documents, key=lambda x: x.metadata['score'], reverse=True)
+        return filter_documents[:top_k] if top_k else filter_documents
+
+
+

api/core/tools/tool/dataset_retriever/dataset_multi_retriever_tool.py

@@ -7,7 +7,7 @@ from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCa
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.rag.datasource.retrieval_service import RetrievalService
-from core.rag.rerank.rerank import RerankRunner
+from core.rag.rerank.rerank_model import RerankModelRunner
 from core.rag.retrieval.retrival_methods import RetrievalMethod
 from core.tools.tool.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseTool
 from extensions.ext_database import db
@@ -72,7 +72,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):
             model=self.reranking_model_name
         )
 
-        rerank_runner = RerankRunner(rerank_model_instance)
+        rerank_runner = RerankModelRunner(rerank_model_instance)
         all_documents = rerank_runner.run(query, all_documents, self.score_threshold, self.top_k)
 
         for hit_callback in self.hit_callbacks:
@@ -180,7 +180,8 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):
                                                           score_threshold=retrieval_model['score_threshold']
                                                           if retrieval_model['score_threshold_enabled'] else None,
                                                           reranking_model=retrieval_model['reranking_model']
-                                                          if retrieval_model['reranking_enable'] else None
+                                                          if retrieval_model['reranking_enable'] else None,
+                                                          weights=retrieval_model.get('weights', None),
                                                           )
 
                     all_documents.extend(documents)

api/core/tools/tool/dataset_retriever/dataset_retriever_tool.py

@@ -78,7 +78,8 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool):
                                                       score_threshold=retrieval_model['score_threshold']
                                                       if retrieval_model['score_threshold_enabled'] else None,
                                                       reranking_model=retrieval_model['reranking_model']
-                                                      if retrieval_model['reranking_enable'] else None
+                                                      if retrieval_model['reranking_enable'] else None,
+                                                      weights=retrieval_model.get('weights', None),
                                                       )
             else:
                 documents = []

api/core/workflow/nodes/knowledge_retrieval/entities.py

@@ -13,13 +13,41 @@ class RerankingModelConfig(BaseModel):
     model: str
 
 
+class VectorSetting(BaseModel):
+    """
+    Vector Setting.
+    """
+    vector_weight: float
+    embedding_provider_name: str
+    embedding_model_name: str
+
+
+class KeywordSetting(BaseModel):
+    """
+    Keyword Setting.
+    """
+    keyword_weight: float
+
+
+class WeightedScoreConfig(BaseModel):
+    """
+    Weighted score Config.
+    """
+    weight_type: str
+    vector_setting: VectorSetting
+    keyword_setting: KeywordSetting
+
+
 class MultipleRetrievalConfig(BaseModel):
     """
     Multiple Retrieval Config.
     """
     top_k: int
     score_threshold: Optional[float] = None
+    reranking_mode: str = 'reranking_model'
+    reranking_enable: bool = True
     reranking_model: RerankingModelConfig
+    weights: WeightedScoreConfig
 
 
 class ModelConfig(BaseModel):

api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py

@@ -138,13 +138,38 @@ class KnowledgeRetrievalNode(BaseNode):
                     planning_strategy=planning_strategy
                 )
         elif node_data.retrieval_mode == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE.value:
+            if node_data.multiple_retrieval_config.reranking_mode == 'reranking_model':
+                reranking_model = {
+                    'reranking_provider_name': node_data.multiple_retrieval_config.reranking_model['provider'],
+                    'reranking_model_name': node_data.multiple_retrieval_config.reranking_model['name']
+                }
+                weights = None
+            elif node_data.multiple_retrieval_config.reranking_mode == 'weighted_score':
+                reranking_model = None
+                weights = {
+                    'weight_type': node_data.multiple_retrieval_config.weights.weight_type,
+                    'vector_setting': {
+                        "vector_weight": node_data.multiple_retrieval_config.weights.vector_setting.vector_weight,
+                        "embedding_provider_name": node_data.multiple_retrieval_config.weights.vector_setting.embedding_provider_name,
+                        "embedding_model_name": node_data.multiple_retrieval_config.weights.vector_setting.embedding_model_name,
+                    },
+                    'keyword_setting': {
+                        "keyword_weight": node_data.multiple_retrieval_config.weights.keyword_setting.keyword_weight
+                    }
+                }
+            else:
+                reranking_model = None
+                weights = None
             all_documents = dataset_retrieval.multiple_retrieve(self.app_id, self.tenant_id, self.user_id,
                                                                 self.user_from.value,
                                                                 available_datasets, query,
                                                                 node_data.multiple_retrieval_config.top_k,
                                                                 node_data.multiple_retrieval_config.score_threshold,
-                                                                node_data.multiple_retrieval_config.reranking_model.provider,
-                                                                node_data.multiple_retrieval_config.reranking_model.model)
+                                                                node_data.multiple_retrieval_config.reranking_mode,
+                                                                reranking_model,
+                                                                weights,
+                                                                node_data.multiple_retrieval_config.reranking_enable,
+                                                                )
 
         context_list = []
         if all_documents:

api/fields/dataset_fields.py

@@ -18,10 +18,28 @@ reranking_model_fields = {
     'reranking_model_name': fields.String
 }
 
+keyword_setting_fields = {
+    'keyword_weight': fields.Float
+}
+
+vector_setting_fields = {
+    'vector_weight': fields.Float,
+    'embedding_model_name': fields.String,
+    'embedding_provider_name': fields.String,
+}
+
+weighted_score_fields = {
+    'weight_type': fields.String,
+    'keyword_setting': fields.Nested(keyword_setting_fields),
+    'vector_setting': fields.Nested(vector_setting_fields),
+}
+
 dataset_retrieval_model_fields = {
     'search_method': fields.String,
     'reranking_enable': fields.Boolean,
+    'reranking_mode': fields.String,
     'reranking_model': fields.Nested(reranking_model_fields),
+    'weights': fields.Nested(weighted_score_fields, allow_null=True),
     'top_k': fields.Integer,
     'score_threshold_enabled': fields.Boolean,
     'score_threshold': fields.Float

api/migrations/versions/53bf8af60645_update_model.py

@@ -0,0 +1,41 @@
+"""update model
+
+Revision ID: 53bf8af60645
+Revises: 8e5588e6412e
+Create Date: 2024-07-24 08:06:55.291031
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+import models as models
+
+# revision identifiers, used by Alembic.
+revision = '53bf8af60645'
+down_revision = '8e5588e6412e'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('embeddings', schema=None) as batch_op:
+        batch_op.alter_column('provider_name',
+               existing_type=sa.VARCHAR(length=40),
+               type_=sa.String(length=255),
+               existing_nullable=False,
+               existing_server_default=sa.text("''::character varying"))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('embeddings', schema=None) as batch_op:
+        batch_op.alter_column('provider_name',
+               existing_type=sa.String(length=255),
+               type_=sa.VARCHAR(length=40),
+               existing_nullable=False,
+               existing_server_default=sa.text("''::character varying"))
+
+    # ### end Alembic commands ###

api/models/dataset.py

@@ -640,7 +640,7 @@ class Embedding(db.Model):
     hash = db.Column(db.String(64), nullable=False)
     embedding = db.Column(db.LargeBinary, nullable=False)
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.text('CURRENT_TIMESTAMP(0)'))
-    provider_name = db.Column(db.String(40), nullable=False,
+    provider_name = db.Column(db.String(255), nullable=False,
                               server_default=db.text("''::character varying"))
 
     def set_embedding(self, embedding_data: list[float]):

api/models/model.py

@@ -328,7 +328,9 @@ class AppModelConfig(db.Model):
                 return {'retrieval_model': 'single'}
             else:
                 return dataset_configs
-        return {'retrieval_model': 'single'}
+        return {
+                'retrieval_model': 'multiple',
+            }
 
     @property
     def file_upload_dict(self) -> dict:

api/pyproject.toml

@@ -163,7 +163,8 @@ redis = { version = "~5.0.3", extras = ["hiredis"] }
 replicate = "~0.22.0"
 resend = "~0.7.0"
 safetensors = "~0.4.3"
-sentry-sdk = { version = "~2.8.0", extras = ["flask"] }
+scikit-learn = "^1.5.1"
+sentry-sdk = { version = "~1.44.1", extras = ["flask"] }
 sqlalchemy = "~2.0.29"
 tencentcloud-sdk-python-hunyuan = "~3.0.1158"
 tiktoken = "~0.7.0"
@@ -175,7 +176,7 @@ werkzeug = "~3.0.1"
 xinference-client = "0.9.4"
 yarl = "~1.9.4"
 zhipuai = "1.0.7"
-
+rank-bm25 = "~0.2.2"
 ############################################################
 # Tool dependencies required by tool implementations
 ############################################################
@@ -200,7 +201,7 @@ cloudscraper = "1.2.71"
 ############################################################
 
 [tool.poetry.group.vdb.dependencies]
-chromadb = "~0.5.1"
+chromadb = "0.5.1"
 oracledb = "~2.2.1"
 pgvecto-rs = "0.1.4"
 pgvector = "0.2.5"

api/services/app_service.py

@@ -287,8 +287,12 @@ class AppService:
         """
         db.session.delete(app)
         db.session.commit()
+
         # Trigger asynchronous deletion of app and related data
-        remove_app_and_related_data_task.delay(app.id)
+        remove_app_and_related_data_task.delay(
+            tenant_id=app.tenant_id,
+            app_id=app.id
+        )
 
     def get_app_meta(self, app_model: App) -> dict:
         """

api/services/hit_testing_service.py

@@ -38,14 +38,16 @@ class HitTestingService:
         if not retrieval_model:
             retrieval_model = dataset.retrieval_model if dataset.retrieval_model else default_retrieval_model
 
-        all_documents = RetrievalService.retrieve(retrival_method=retrieval_model['search_method'],
+        all_documents = RetrievalService.retrieve(retrival_method=retrieval_model.get('search_method', 'semantic_search'),
                                                   dataset_id=dataset.id,
                                                   query=cls.escape_query_for_search(query),
-                                                  top_k=retrieval_model['top_k'],
+                                                  top_k=retrieval_model.get('top_k', 2),
                                                   score_threshold=retrieval_model['score_threshold']
                                                   if retrieval_model['score_threshold_enabled'] else None,
                                                   reranking_model=retrieval_model['reranking_model']
-                                                  if retrieval_model['reranking_enable'] else None
+                                                  if retrieval_model['reranking_enable'] else None,
+                                                  reranking_mode=retrieval_model.get('reranking_mode', None),
+                                                  weights=retrieval_model.get('weights', None),
                                                   )
 
         end = time.perf_counter()

api/tasks/remove_app_and_related_data_task.py

@@ -3,7 +3,6 @@ import time
 
 import click
 from celery import shared_task
-from sqlalchemy import select
 from sqlalchemy.exc import SQLAlchemyError
 
 from extensions.ext_database import db
@@ -25,6 +24,7 @@ from models.model import (
     RecommendedApp,
     Site,
     TagBinding,
+    TraceAppConfig,
 )
 from models.tools import WorkflowToolProvider
 from models.web import PinnedConversation, SavedMessage
@@ -32,122 +32,287 @@ from models.workflow import Workflow, WorkflowAppLog, WorkflowNodeExecution, Wor
 
 
 @shared_task(queue='app_deletion', bind=True, max_retries=3)
-def remove_app_and_related_data_task(self, app_id: str):
-    logging.info(click.style(f'Start deleting app and related data: {app_id}', fg='green'))
+def remove_app_and_related_data_task(self, tenant_id: str, app_id: str):
+    logging.info(click.style(f'Start deleting app and related data: {tenant_id}:{app_id}', fg='green'))
     start_at = time.perf_counter()
     try:
-        # Use a transaction to ensure all deletions succeed or none do
-        with db.session.begin_nested():
-            # Delete related data
-            _delete_app_model_configs(app_id)
-            _delete_app_site(app_id)
-            _delete_app_api_tokens(app_id)
-            _delete_installed_apps(app_id)
-            _delete_recommended_apps(app_id)
-            _delete_app_annotation_data(app_id)
-            _delete_app_dataset_joins(app_id)
-            _delete_app_workflows(app_id)
-            _delete_app_conversations(app_id)
-            _delete_app_messages(app_id)
-            _delete_workflow_tool_providers(app_id)
-            _delete_app_tag_bindings(app_id)
-            _delete_end_users(app_id)
-
-        # If we reach here, the transaction was successful
-        db.session.commit()
+        # Delete related data
+        _delete_app_model_configs(tenant_id, app_id)
+        _delete_app_site(tenant_id, app_id)
+        _delete_app_api_tokens(tenant_id, app_id)
+        _delete_installed_apps(tenant_id, app_id)
+        _delete_recommended_apps(tenant_id, app_id)
+        _delete_app_annotation_data(tenant_id, app_id)
+        _delete_app_dataset_joins(tenant_id, app_id)
+        _delete_app_workflows(tenant_id, app_id)
+        _delete_app_workflow_runs(tenant_id, app_id)
+        _delete_app_workflow_node_executions(tenant_id, app_id)
+        _delete_app_workflow_app_logs(tenant_id, app_id)
+        _delete_app_conversations(tenant_id, app_id)
+        _delete_app_messages(tenant_id, app_id)
+        _delete_workflow_tool_providers(tenant_id, app_id)
+        _delete_app_tag_bindings(tenant_id, app_id)
+        _delete_end_users(tenant_id, app_id)
+        _delete_trace_app_configs(tenant_id, app_id)
 
         end_at = time.perf_counter()
         logging.info(click.style(f'App and related data deleted: {app_id} latency: {end_at - start_at}', fg='green'))
-
     except SQLAlchemyError as e:
-        db.session.rollback()
         logging.exception(
             click.style(f"Database error occurred while deleting app {app_id} and related data", fg='red'))
         raise self.retry(exc=e, countdown=60)  # Retry after 60 seconds
-
     except Exception as e:
         logging.exception(click.style(f"Error occurred while deleting app {app_id} and related data", fg='red'))
         raise self.retry(exc=e, countdown=60)  # Retry after 60 seconds
 
 
-def _delete_app_model_configs(app_id: str):
-    db.session.query(AppModelConfig).filter(AppModelConfig.app_id == app_id).delete()
+def _delete_app_model_configs(tenant_id: str, app_id: str):
+    def del_model_config(model_config_id: str):
+        db.session.query(AppModelConfig).filter(AppModelConfig.id == model_config_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from app_model_configs where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_model_config,
+        "app model config"
+    )
 
 
-def _delete_app_site(app_id: str):
-    db.session.query(Site).filter(Site.app_id == app_id).delete()
+def _delete_app_site(tenant_id: str, app_id: str):
+    def del_site(site_id: str):
+        db.session.query(Site).filter(Site.id == site_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from sites where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_site,
+        "site"
+    )
 
 
-def _delete_app_api_tokens(app_id: str):
-    db.session.query(ApiToken).filter(ApiToken.app_id == app_id).delete()
+def _delete_app_api_tokens(tenant_id: str, app_id: str):
+    def del_api_token(api_token_id: str):
+        db.session.query(ApiToken).filter(ApiToken.id == api_token_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from api_tokens where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_api_token,
+        "api token"
+    )
 
 
-def _delete_installed_apps(app_id: str):
-    db.session.query(InstalledApp).filter(InstalledApp.app_id == app_id).delete()
+def _delete_installed_apps(tenant_id: str, app_id: str):
+    def del_installed_app(installed_app_id: str):
+        db.session.query(InstalledApp).filter(InstalledApp.id == installed_app_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from installed_apps where tenant_id=:tenant_id and app_id=:app_id limit 1000""",
+        {"tenant_id": tenant_id, "app_id": app_id},
+        del_installed_app,
+        "installed app"
+    )
 
 
-def _delete_recommended_apps(app_id: str):
-    db.session.query(RecommendedApp).filter(RecommendedApp.app_id == app_id).delete()
+def _delete_recommended_apps(tenant_id: str, app_id: str):
+    def del_recommended_app(recommended_app_id: str):
+        db.session.query(RecommendedApp).filter(RecommendedApp.id == recommended_app_id).delete(
+            synchronize_session=False)
+
+    _delete_records(
+        """select id from recommended_apps where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_recommended_app,
+        "recommended app"
+    )
 
 
-def _delete_app_annotation_data(app_id: str):
-    db.session.query(AppAnnotationHitHistory).filter(AppAnnotationHitHistory.app_id == app_id).delete()
-    db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.app_id == app_id).delete()
+def _delete_app_annotation_data(tenant_id: str, app_id: str):
+    def del_annotation_hit_history(annotation_hit_history_id: str):
+        db.session.query(AppAnnotationHitHistory).filter(
+            AppAnnotationHitHistory.id == annotation_hit_history_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from app_annotation_hit_histories where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_annotation_hit_history,
+        "annotation hit history"
+    )
+
+    def del_annotation_setting(annotation_setting_id: str):
+        db.session.query(AppAnnotationSetting).filter(AppAnnotationSetting.id == annotation_setting_id).delete(
+            synchronize_session=False)
+
+    _delete_records(
+        """select id from app_annotation_settings where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_annotation_setting,
+        "annotation setting"
+    )
 
 
-def _delete_app_dataset_joins(app_id: str):
-    db.session.query(AppDatasetJoin).filter(AppDatasetJoin.app_id == app_id).delete()
+def _delete_app_dataset_joins(tenant_id: str, app_id: str):
+    def del_dataset_join(dataset_join_id: str):
+        db.session.query(AppDatasetJoin).filter(AppDatasetJoin.id == dataset_join_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from app_dataset_joins where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_dataset_join,
+        "dataset join"
+    )
 
 
-def _delete_app_workflows(app_id: str):
-    db.session.query(WorkflowRun).filter(
-        WorkflowRun.workflow_id.in_(
-            db.session.query(Workflow.id).filter(Workflow.app_id == app_id)
-        )
-    ).delete(synchronize_session=False)
-    db.session.query(WorkflowNodeExecution).filter(
-        WorkflowNodeExecution.workflow_id.in_(
-            db.session.query(Workflow.id).filter(Workflow.app_id == app_id)
-        )
-    ).delete(synchronize_session=False)
-    db.session.query(WorkflowAppLog).filter(WorkflowAppLog.app_id == app_id).delete(synchronize_session=False)
-    db.session.query(Workflow).filter(Workflow.app_id == app_id).delete(synchronize_session=False)
+def _delete_app_workflows(tenant_id: str, app_id: str):
+    def del_workflow(workflow_id: str):
+        db.session.query(Workflow).filter(Workflow.id == workflow_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from workflows where tenant_id=:tenant_id and app_id=:app_id limit 1000""",
+        {"tenant_id": tenant_id, "app_id": app_id},
+        del_workflow,
+        "workflow"
+    )
 
 
-def _delete_app_conversations(app_id: str):
-    db.session.query(PinnedConversation).filter(
-        PinnedConversation.conversation_id.in_(
-            db.session.query(Conversation.id).filter(Conversation.app_id == app_id)
-        )
-    ).delete(synchronize_session=False)
-    db.session.query(Conversation).filter(Conversation.app_id == app_id).delete()
+def _delete_app_workflow_runs(tenant_id: str, app_id: str):
+    def del_workflow_run(workflow_run_id: str):
+        db.session.query(WorkflowRun).filter(WorkflowRun.id == workflow_run_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from workflow_runs where tenant_id=:tenant_id and app_id=:app_id limit 1000""",
+        {"tenant_id": tenant_id, "app_id": app_id},
+        del_workflow_run,
+        "workflow run"
+    )
 
 
-def _delete_app_messages(app_id: str):
-    message_ids = select(Message.id).filter(Message.app_id == app_id).scalar_subquery()
-    db.session.query(MessageFeedback).filter(MessageFeedback.message_id.in_(message_ids)).delete(
-        synchronize_session=False)
-    db.session.query(MessageAnnotation).filter(MessageAnnotation.message_id.in_(message_ids)).delete(
-        synchronize_session=False)
-    db.session.query(MessageChain).filter(MessageChain.message_id.in_(message_ids)).delete(synchronize_session=False)
-    db.session.query(MessageAgentThought).filter(MessageAgentThought.message_id.in_(message_ids)).delete(
-        synchronize_session=False)
-    db.session.query(MessageFile).filter(MessageFile.message_id.in_(message_ids)).delete(synchronize_session=False)
-    db.session.query(SavedMessage).filter(SavedMessage.message_id.in_(message_ids)).delete(synchronize_session=False)
-    db.session.query(Message).filter(Message.app_id == app_id).delete(synchronize_session=False)
+def _delete_app_workflow_node_executions(tenant_id: str, app_id: str):
+    def del_workflow_node_execution(workflow_node_execution_id: str):
+        db.session.query(WorkflowNodeExecution).filter(
+            WorkflowNodeExecution.id == workflow_node_execution_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from workflow_node_executions where tenant_id=:tenant_id and app_id=:app_id limit 1000""",
+        {"tenant_id": tenant_id, "app_id": app_id},
+        del_workflow_node_execution,
+        "workflow node execution"
+    )
 
 
-def _delete_workflow_tool_providers(app_id: str):
-    db.session.query(WorkflowToolProvider).filter(
-        WorkflowToolProvider.app_id == app_id
-    ).delete(synchronize_session=False)
+def _delete_app_workflow_app_logs(tenant_id: str, app_id: str):
+    def del_workflow_app_log(workflow_app_log_id: str):
+        db.session.query(WorkflowAppLog).filter(WorkflowAppLog.id == workflow_app_log_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from workflow_app_logs where tenant_id=:tenant_id and app_id=:app_id limit 1000""",
+        {"tenant_id": tenant_id, "app_id": app_id},
+        del_workflow_app_log,
+        "workflow app log"
+    )
 
 
-def _delete_app_tag_bindings(app_id: str):
-    db.session.query(TagBinding).filter(
-        TagBinding.target_id == app_id
-    ).delete(synchronize_session=False)
+def _delete_app_conversations(tenant_id: str, app_id: str):
+    def del_conversation(conversation_id: str):
+        db.session.query(PinnedConversation).filter(PinnedConversation.conversation_id == conversation_id).delete(
+            synchronize_session=False)
+        db.session.query(Conversation).filter(Conversation.id == conversation_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from conversations where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_conversation,
+        "conversation"
+    )
 
 
-def _delete_end_users(app_id: str):
-    db.session.query(EndUser).filter(EndUser.app_id == app_id).delete()
+def _delete_app_messages(tenant_id: str, app_id: str):
+    def del_message(message_id: str):
+        db.session.query(MessageFeedback).filter(MessageFeedback.message_id == message_id).delete(
+            synchronize_session=False)
+        db.session.query(MessageAnnotation).filter(MessageAnnotation.message_id == message_id).delete(
+            synchronize_session=False)
+        db.session.query(MessageChain).filter(MessageChain.message_id == message_id).delete(
+            synchronize_session=False)
+        db.session.query(MessageAgentThought).filter(MessageAgentThought.message_id == message_id).delete(
+            synchronize_session=False)
+        db.session.query(MessageFile).filter(MessageFile.message_id == message_id).delete(synchronize_session=False)
+        db.session.query(SavedMessage).filter(SavedMessage.message_id == message_id).delete(
+            synchronize_session=False)
+        db.session.query(Message).filter(Message.id == message_id).delete()
+
+    _delete_records(
+        """select id from messages where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_message,
+        "message"
+    )
+
+
+def _delete_workflow_tool_providers(tenant_id: str, app_id: str):
+    def del_tool_provider(tool_provider_id: str):
+        db.session.query(WorkflowToolProvider).filter(WorkflowToolProvider.id == tool_provider_id).delete(
+            synchronize_session=False)
+
+    _delete_records(
+        """select id from tool_workflow_providers where tenant_id=:tenant_id and app_id=:app_id limit 1000""",
+        {"tenant_id": tenant_id, "app_id": app_id},
+        del_tool_provider,
+        "tool workflow provider"
+    )
+
+
+def _delete_app_tag_bindings(tenant_id: str, app_id: str):
+    def del_tag_binding(tag_binding_id: str):
+        db.session.query(TagBinding).filter(TagBinding.id == tag_binding_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from tag_bindings where tenant_id=:tenant_id and target_id=:app_id limit 1000""",
+        {"tenant_id": tenant_id, "app_id": app_id},
+        del_tag_binding,
+        "tag binding"
+    )
+
+
+def _delete_end_users(tenant_id: str, app_id: str):
+    def del_end_user(end_user_id: str):
+        db.session.query(EndUser).filter(EndUser.id == end_user_id).delete(synchronize_session=False)
+
+    _delete_records(
+        """select id from end_users where tenant_id=:tenant_id and app_id=:app_id limit 1000""",
+        {"tenant_id": tenant_id, "app_id": app_id},
+        del_end_user,
+        "end user"
+    )
+
+
+def _delete_trace_app_configs(tenant_id: str, app_id: str):
+    def del_trace_app_config(trace_app_config_id: str):
+        db.session.query(TraceAppConfig).filter(TraceAppConfig.id == trace_app_config_id).delete(
+            synchronize_session=False)
+
+    _delete_records(
+        """select id from trace_app_config where app_id=:app_id limit 1000""",
+        {"app_id": app_id},
+        del_trace_app_config,
+        "trace app config"
+    )
+
+
+def _delete_records(query_sql: str, params: dict, delete_func: callable, name: str) -> None:
+    while True:
+        with db.engine.begin() as conn:
+            rs = conn.execute(db.text(query_sql), params)
+            if rs.rowcount == 0:
+                break
+
+            for i in rs:
+                record_id = str(i.id)
+                try:
+                    delete_func(record_id)
+                    db.session.commit()
+                    logging.info(click.style(f"Deleted {name} {record_id}", fg='green'))
+                except Exception:
+                    logging.exception(f"Error occurred while deleting {name} {record_id}")
+                    continue
+            rs.close()

dev/pytest/pytest_vdb.sh

@@ -3,7 +3,6 @@ set -x
 
 pytest api/tests/integration_tests/vdb/chroma \
   api/tests/integration_tests/vdb/milvus \
-  api/tests/integration_tests/vdb/myscale \
   api/tests/integration_tests/vdb/pgvecto_rs \
   api/tests/integration_tests/vdb/pgvector \
   api/tests/integration_tests/vdb/qdrant \

web/app/components/app-sidebar/app-info.tsx

@@ -434,7 +434,7 @@ const AppInfo = ({ expand }: IAppInfoProps) => {
         {showImportDSLModal && (
           <UpdateDSLModal
             onCancel={() => setShowImportDSLModal(false)}
-            onBackup={onExport}
+            onBackup={exportCheck}
           />
         )}
         {secretEnvList.length > 0 && (

web/app/components/app/configuration/dataset-config/card-item/item.tsx

@@ -1,18 +1,20 @@
 'use client'
 import type { FC } from 'react'
 import React, { useState } from 'react'
-import { useTranslation } from 'react-i18next'
-import { RiDeleteBinLine } from '@remixicon/react'
+import {
+  RiDeleteBinLine,
+  RiEditLine,
+} from '@remixicon/react'
 import SettingsModal from '../settings-modal'
 import type { DataSet } from '@/models/datasets'
 import { DataSourceType } from '@/models/datasets'
-import { formatNumber } from '@/utils/format'
 import FileIcon from '@/app/components/base/file-icon'
-import { Settings01 } from '@/app/components/base/icons/src/vender/line/general'
 import { Folder } from '@/app/components/base/icons/src/vender/solid/files'
 import { Globe06 } from '@/app/components/base/icons/src/vender/solid/mapsAndTravel'
 import Drawer from '@/app/components/base/drawer'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
+import Badge from '@/app/components/base/badge'
+import { useKnowledge } from '@/hooks/use-knowledge'
 
 type ItemProps = {
   className?: string
@@ -27,12 +29,10 @@ const Item: FC<ItemProps> = ({
   onSave,
   onRemove,
 }) => {
-  const { t } = useTranslation()
-
   const media = useBreakpoints()
   const isMobile = media === MediaType.mobile
-
   const [showSettingsModal, setShowSettingsModal] = useState(false)
+  const { formatIndexingTechniqueAndMethod } = useKnowledge()
 
   const handleSave = (newDataset: DataSet) => {
     onSave(newDataset)
@@ -65,22 +65,17 @@ const Item: FC<ItemProps> = ({
       <div className='grow'>
         <div className='flex items-center h-[18px]'>
           <div className='grow text-[13px] font-medium text-gray-800 truncate' title={config.name}>{config.name}</div>
-          <div className='shrink-0 text-xs text-gray-500'>
-            {formatNumber(config.word_count)} {t('appDebug.feature.dataSet.words')} · {formatNumber(config.document_count)} {t('appDebug.feature.dataSet.textBlocks')}
-          </div>
+          <Badge
+            text={formatIndexingTechniqueAndMethod(config.indexing_technique, config.retrieval_model_dict?.search_method)}
+          />
         </div>
-        {/* {
-          config.description && (
-            <div className='text-xs text-gray-500'>{config.description}</div>
-          )
-        } */}
       </div>
       <div className='hidden rounded-lg group-hover:flex items-center justify-end absolute right-0 top-0 bottom-0 pr-2 w-[124px] bg-gradient-to-r from-white/50 to-white to-50%'>
         <div
           className='flex items-center justify-center mr-1 w-6 h-6 hover:bg-black/5 rounded-md cursor-pointer'
           onClick={() => setShowSettingsModal(true)}
         >
-          <Settings01 className='w-4 h-4 text-gray-500' />
+          <RiEditLine className='w-4 h-4 text-gray-500' />
         </div>
         <div
           className='group/action flex items-center justify-center w-6 h-6 hover:bg-[#FEE4E2] rounded-md cursor-pointer'

web/app/components/app/configuration/dataset-config/index.tsx

@@ -44,7 +44,8 @@ const DatasetConfig: FC = () => {
   const handleSave = (newDataset: DataSet) => {
     const index = dataSet.findIndex(item => item.id === newDataset.id)
 
-    setDataSet([...dataSet.slice(0, index), newDataset, ...dataSet.slice(index + 1)])
+    const newDatasets = [...dataSet.slice(0, index), newDataset, ...dataSet.slice(index + 1)]
+    setDataSet(newDatasets)
     formattingChangedDispatcher()
   }
 
@@ -74,7 +75,7 @@ const DatasetConfig: FC = () => {
       title={t('appDebug.feature.dataSet.title')}
       headerRight={
         <div className='flex items-center gap-1'>
-          {!isAgent && <ParamsConfig />}
+          {!isAgent && <ParamsConfig disabled={!hasData} selectedDatasets={dataSet} />}
           <OperationBtn type="add" onClick={showSelectDataSet} />
         </div>
       }

web/app/components/app/configuration/dataset-config/params-config/config-content.tsx

@@ -1,10 +1,12 @@
 'use client'
-import React from 'react'
+
+import { memo, useMemo } from 'react'
 import type { FC } from 'react'
 import { useTranslation } from 'react-i18next'
 import {
   RiQuestionLine,
 } from '@remixicon/react'
+import WeightedScore from './weighted-score'
 import TopKItem from '@/app/components/base/param-item/top-k-item'
 import ScoreThresholdItem from '@/app/components/base/param-item/score-threshold-item'
 import RadioCard from '@/app/components/base/radio-card/simple'
@@ -16,13 +18,20 @@ import {
 import type {
   DatasetConfigs,
 } from '@/models/debug'
-
 import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
 import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import type { ModelConfig } from '@/app/components/workflow/types'
 import ModelParameterModal from '@/app/components/header/account-setting/model-provider-page/model-parameter-modal'
 import TooltipPlus from '@/app/components/base/tooltip-plus'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import type {
+  DataSet,
+  WeightedScoreEnum,
+} from '@/models/datasets'
+import { RerankingModeEnum } from '@/models/datasets'
+import cn from '@/utils/classnames'
+import { useSelectedDatasetsMode } from '@/app/components/workflow/nodes/knowledge-retrieval/hooks'
+import Switch from '@/app/components/base/switch'
 
 type Props = {
   datasetConfigs: DatasetConfigs
@@ -31,6 +40,7 @@ type Props = {
   singleRetrievalModelConfig?: ModelConfig
   onSingleRetrievalModelChange?: (config: ModelConfig) => void
   onSingleRetrievalModelParamsChange?: (config: ModelConfig) => void
+  selectedDatasets?: DataSet[]
 }
 
 const ConfigContent: FC<Props> = ({
@@ -40,8 +50,10 @@ const ConfigContent: FC<Props> = ({
   singleRetrievalModelConfig: singleRetrievalConfig = {} as ModelConfig,
   onSingleRetrievalModelChange = () => { },
   onSingleRetrievalModelParamsChange = () => { },
+  selectedDatasets = [],
 }) => {
   const { t } = useTranslation()
+  const selectedDatasetsMode = useSelectedDatasetsMode(selectedDatasets)
   const type = datasetConfigs.retrieval_model
   const setType = (value: RETRIEVE_TYPE) => {
     onChange({
@@ -54,7 +66,7 @@ const ConfigContent: FC<Props> = ({
     defaultModel: rerankDefaultModel,
   } = useModelListAndDefaultModelAndCurrentProviderAndModel(ModelTypeEnum.rerank)
   const rerankModel = (() => {
-    if (datasetConfigs.reranking_model) {
+    if (datasetConfigs.reranking_model?.reranking_provider_name) {
       return {
         provider_name: datasetConfigs.reranking_model.reranking_provider_name,
         model_name: datasetConfigs.reranking_model.reranking_model_name,
@@ -93,14 +105,73 @@ const ConfigContent: FC<Props> = ({
     })
   }
 
+  const handleWeightedScoreChange = (value: { type: WeightedScoreEnum; value: number[] }) => {
+    const configs = {
+      ...datasetConfigs,
+      weights: {
+        ...datasetConfigs.weights!,
+        weight_type: value.type,
+        vector_setting: {
+          ...datasetConfigs.weights!.vector_setting!,
+          vector_weight: value.value[0],
+        },
+        keyword_setting: {
+          keyword_weight: value.value[1],
+        },
+      },
+    }
+    onChange(configs)
+  }
+
+  const handleRerankModeChange = (mode: RerankingModeEnum) => {
+    onChange({
+      ...datasetConfigs,
+      reranking_mode: mode,
+    })
+  }
+
   const model = singleRetrievalConfig
 
+  const rerankingModeOptions = [
+    {
+      value: RerankingModeEnum.WeightedScore,
+      label: t('dataset.weightedScore.title'),
+      tips: t('dataset.weightedScore.description'),
+    },
+    {
+      value: RerankingModeEnum.RerankingModel,
+      label: t('common.modelProvider.rerankModel.key'),
+      tips: t('common.modelProvider.rerankModel.tip'),
+    },
+  ]
+
+  const showWeightedScore = selectedDatasetsMode.allHighQuality
+    && !selectedDatasetsMode.inconsistentEmbeddingModel
+
+  const showWeightedScorePanel = showWeightedScore && datasetConfigs.reranking_mode === RerankingModeEnum.WeightedScore && datasetConfigs.weights
+  const selectedRerankMode = datasetConfigs.reranking_mode || RerankingModeEnum.RerankingModel
+
+  const showRerankModel = useMemo(() => {
+    if (datasetConfigs.reranking_enable === false && selectedDatasetsMode.allEconomic)
+      return false
+
+    return true
+  }, [datasetConfigs.reranking_enable, selectedDatasetsMode.allEconomic])
+
   return (
     <div>
+      <div className='system-xl-semibold text-text-primary'>{t('dataset.retrievalSettings')}</div>
       <div className='mt-2 space-y-3'>
         <RadioCard
           icon={<NTo1Retrieval className='shrink-0 mr-3 w-9 h-9 rounded-lg' />}
-          title={t('appDebug.datasetConfig.retrieveOneWay.title')}
+          title={(
+            <div className='flex items-center'>
+              {t('appDebug.datasetConfig.retrieveOneWay.title')}
+              <TooltipPlus popupContent={<div className='w-[320px]'>{t('dataset.nTo1RetrievalLegacy')}</div>}>
+                <div className='ml-1 flex items-center px-[5px] h-[18px] rounded-[5px] border border-text-accent-secondary system-2xs-medium-uppercase text-text-accent-secondary'>legacy</div>
+              </TooltipPlus>
+            </div>
+          )}
           description={t('appDebug.datasetConfig.retrieveOneWay.description')}
           isChosen={type === RETRIEVE_TYPE.oneWay}
           onChosen={() => { setType(RETRIEVE_TYPE.oneWay) }}
@@ -115,43 +186,152 @@ const ConfigContent: FC<Props> = ({
       </div>
       {type === RETRIEVE_TYPE.multiWay && (
         <>
-          <div className='mt-6'>
-            <div className='leading-[32px] text-[13px] font-medium text-gray-900'>{t('common.modelProvider.rerankModel.key')}</div>
-            <div>
-              <ModelSelector
-                defaultModel={rerankModel && { provider: rerankModel?.provider_name, model: rerankModel?.model_name }}
-                onSelect={(v) => {
-                  onChange({
-                    ...datasetConfigs,
-                    reranking_model: {
-                      reranking_provider_name: v.provider,
-                      reranking_model_name: v.model,
-                    },
-                  })
-                }}
-                modelList={rerankModelList}
-              />
-            </div>
-          </div>
-          <div className='mt-4 space-y-4'>
-            <TopKItem
-              value={datasetConfigs.top_k}
-              onChange={handleParamChange}
-              enable={true}
-            />
-            <ScoreThresholdItem
-              value={datasetConfigs.score_threshold as number}
-              onChange={handleParamChange}
-              enable={datasetConfigs.score_threshold_enabled}
-              hasSwitch={true}
-              onSwitchChange={handleSwitch}
-            />
+          <div className='mb-2 mt-4 h-[1px] bg-divider-subtle'></div>
+          <div
+            className='flex items-center mb-2 h-6 system-md-semibold text-text-secondary'
+          >
+            {t('dataset.rerankSettings')}
           </div>
+          {
+            selectedDatasetsMode.inconsistentEmbeddingModel
+            && (
+              <div className='mt-4 system-xs-regular text-text-warning'>
+                {t('dataset.inconsistentEmbeddingModelTip')}
+              </div>
+            )
+          }
+          {
+            selectedDatasetsMode.mixtureHighQualityAndEconomic
+            && (
+              <div className='mt-4 system-xs-regular text-text-warning'>
+                {t('dataset.mixtureHighQualityAndEconomicTip')}
+              </div>
+            )
+          }
+          {
+            showWeightedScore && (
+              <div className='flex items-center justify-between'>
+                {
+                  rerankingModeOptions.map(option => (
+                    <div
+                      key={option.value}
+                      className={cn(
+                        'flex items-center justify-center w-[calc((100%-8px)/2)] h-8 rounded-lg border border-components-option-card-option-border bg-components-option-card-option-bg cursor-pointer system-sm-medium text-text-secondary',
+                        selectedRerankMode === option.value && 'border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary',
+                      )}
+                      onClick={() => handleRerankModeChange(option.value)}
+                    >
+                      <div className='truncate'>{option.label}</div>
+                      <TooltipPlus
+                        popupContent={<div className='w-[200px]'>{option.tips}</div>}
+                        hideArrow
+                      >
+                        <RiQuestionLine className='ml-0.5 w-3.5 h-4.5 text-text-quaternary' />
+                      </TooltipPlus>
+                    </div>
+                  ))
+                }
+              </div>
+            )
+          }
+          {
+            !showWeightedScorePanel && (
+              <div className='mt-2'>
+                <div className='flex items-center'>
+                  {
+                    selectedDatasetsMode.allEconomic && (
+                      <Switch
+                        size='md'
+                        defaultValue={showRerankModel}
+                        onChange={(v) => {
+                          onChange({
+                            ...datasetConfigs,
+                            reranking_enable: v,
+                          })
+                        }}
+                      />
+                    )
+                  }
+                  <div className='ml-2 leading-[32px] text-[13px] font-medium text-gray-900'>{t('common.modelProvider.rerankModel.key')}</div>
+                  <TooltipPlus popupContent={<div className="w-[200px]">{t('common.modelProvider.rerankModel.tip')}</div>}>
+                    <RiQuestionLine className='ml-0.5 w-[14px] h-[14px] text-gray-400' />
+                  </TooltipPlus>
+                </div>
+                <div>
+                  <ModelSelector
+                    defaultModel={rerankModel && { provider: rerankModel?.provider_name, model: rerankModel?.model_name }}
+                    onSelect={(v) => {
+                      onChange({
+                        ...datasetConfigs,
+                        reranking_model: {
+                          reranking_provider_name: v.provider,
+                          reranking_model_name: v.model,
+                        },
+                      })
+                    }}
+                    modelList={rerankModelList}
+                  />
+                </div>
+              </div>
+            )
+          }
+          {
+            showWeightedScorePanel
+            && (
+              <div className='mt-2 space-y-4'>
+                <WeightedScore
+                  value={{
+                    type: datasetConfigs.weights!.weight_type,
+                    value: [
+                      datasetConfigs.weights!.vector_setting.vector_weight,
+                      datasetConfigs.weights!.keyword_setting.keyword_weight,
+                    ],
+                  }}
+                  onChange={handleWeightedScoreChange}
+                />
+                <TopKItem
+                  value={datasetConfigs.top_k}
+                  onChange={handleParamChange}
+                  enable={true}
+                />
+                <ScoreThresholdItem
+                  value={datasetConfigs.score_threshold as number}
+                  onChange={handleParamChange}
+                  enable={datasetConfigs.score_threshold_enabled}
+                  hasSwitch={true}
+                  onSwitchChange={handleSwitch}
+                />
+              </div>
+            )
+          }
+          {
+            !showWeightedScorePanel
+            && (
+              <div className='mt-4 space-y-4'>
+                <TopKItem
+                  value={datasetConfigs.top_k}
+                  onChange={handleParamChange}
+                  enable={true}
+                />
+                {
+                  showRerankModel && (
+                    <ScoreThresholdItem
+                      value={datasetConfigs.score_threshold as number}
+                      onChange={handleParamChange}
+                      enable={datasetConfigs.score_threshold_enabled}
+                      hasSwitch={true}
+                      onSwitchChange={handleSwitch}
+                    />
+                  )
+                }
+              </div>
+            )
+          }
         </>
       )}
 
       {isInWorkflow && type === RETRIEVE_TYPE.oneWay && (
-        <div className='mt-6'>
+        <div className='mt-4'>
           <div className='flex items-center space-x-0.5'>
             <div className='leading-[32px] text-[13px] font-medium text-gray-900'>{t('common.modelProvider.systemReasoningModel.key')}</div>
             <TooltipPlus
@@ -180,4 +360,4 @@ const ConfigContent: FC<Props> = ({
     </div >
   )
 }
-export default React.memo(ConfigContent)
+export default memo(ConfigContent)

web/app/components/app/configuration/dataset-config/params-config/index.tsx

@@ -1,29 +1,73 @@
 'use client'
-import type { FC } from 'react'
-import { memo, useState } from 'react'
+import { memo, useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
+import { RiEqualizer2Line } from '@remixicon/react'
 import ConfigContent from './config-content'
 import cn from '@/utils/classnames'
-import { Settings04 } from '@/app/components/base/icons/src/vender/line/general'
 import ConfigContext from '@/context/debug-configuration'
 import Modal from '@/app/components/base/modal'
 import Button from '@/app/components/base/button'
 import { RETRIEVE_TYPE } from '@/types/app'
 import Toast from '@/app/components/base/toast'
-import { DATASET_DEFAULT } from '@/config'
 import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import type { DataSet } from '@/models/datasets'
+import type { DatasetConfigs } from '@/models/debug'
+import {
+  getMultipleRetrievalConfig,
+  getSelectedDatasetsMode,
+} from '@/app/components/workflow/nodes/knowledge-retrieval/utils'
 
-const ParamsConfig: FC = () => {
+type ParamsConfigProps = {
+  disabled?: boolean
+  selectedDatasets: DataSet[]
+}
+const ParamsConfig = ({
+  disabled,
+  selectedDatasets,
+}: ParamsConfigProps) => {
   const { t } = useTranslation()
-  const [open, setOpen] = useState(false)
   const {
     datasetConfigs,
     setDatasetConfigs,
+    rerankSettingModalOpen,
+    setRerankSettingModalOpen,
   } = useContext(ConfigContext)
   const [tempDataSetConfigs, setTempDataSetConfigs] = useState(datasetConfigs)
 
+  useEffect(() => {
+    const {
+      allEconomic,
+    } = getSelectedDatasetsMode(selectedDatasets)
+    const { datasets, retrieval_model, score_threshold_enabled, ...restConfigs } = datasetConfigs
+    let rerankEnable = restConfigs.reranking_enable
+
+    if (allEconomic && !restConfigs.reranking_model?.reranking_provider_name && rerankEnable === undefined)
+      rerankEnable = false
+
+    setTempDataSetConfigs({
+      ...getMultipleRetrievalConfig({
+        top_k: restConfigs.top_k,
+        score_threshold: restConfigs.score_threshold,
+        reranking_model: restConfigs.reranking_model && {
+          provider: restConfigs.reranking_model.reranking_provider_name,
+          model: restConfigs.reranking_model.reranking_model_name,
+        },
+        reranking_mode: restConfigs.reranking_mode,
+        weights: restConfigs.weights,
+        reranking_enable: rerankEnable,
+      }, selectedDatasets),
+      reranking_model: restConfigs.reranking_model && {
+        reranking_provider_name: restConfigs.reranking_model.reranking_provider_name,
+        reranking_model_name: restConfigs.reranking_model.reranking_model_name,
+      },
+      retrieval_model,
+      score_threshold_enabled,
+      datasets,
+    })
+  }, [selectedDatasets, datasetConfigs])
+
   const {
     defaultModel: rerankDefaultModel,
     currentModel: isRerankDefaultModelVaild,
@@ -55,45 +99,68 @@ const ParamsConfig: FC = () => {
       } as any
     }
     setDatasetConfigs(config)
-    setOpen(false)
+    setRerankSettingModalOpen(false)
+  }
+
+  const handleSetTempDataSetConfigs = (newDatasetConfigs: DatasetConfigs) => {
+    const { datasets, retrieval_model, score_threshold_enabled, ...restConfigs } = newDatasetConfigs
+
+    const retrievalConfig = getMultipleRetrievalConfig({
+      top_k: restConfigs.top_k,
+      score_threshold: restConfigs.score_threshold,
+      reranking_model: restConfigs.reranking_model && {
+        provider: restConfigs.reranking_model.reranking_provider_name,
+        model: restConfigs.reranking_model.reranking_model_name,
+      },
+      reranking_mode: restConfigs.reranking_mode,
+      weights: restConfigs.weights,
+      reranking_enable: restConfigs.reranking_enable,
+    }, selectedDatasets)
+
+    setTempDataSetConfigs({
+      ...retrievalConfig,
+      reranking_model: restConfigs.reranking_model && {
+        reranking_provider_name: restConfigs.reranking_model.reranking_provider_name,
+        reranking_model_name: restConfigs.reranking_model.reranking_model_name,
+      },
+      retrieval_model,
+      score_threshold_enabled,
+      datasets,
+    })
   }
 
   return (
     <div>
-      <div
-        className={cn('flex items-center rounded-md h-7 px-3 space-x-1 text-gray-700 cursor-pointer hover:bg-gray-200', open && 'bg-gray-200')}
+      <Button
+        variant='ghost'
+        size='small'
+        className={cn('h-7', rerankSettingModalOpen && 'bg-components-button-ghost-bg-hover')}
         onClick={() => {
-          setTempDataSetConfigs({
-            ...datasetConfigs,
-            top_k: datasetConfigs.top_k || DATASET_DEFAULT.top_k,
-            score_threshold: datasetConfigs.score_threshold || DATASET_DEFAULT.score_threshold,
-          })
-          setOpen(true)
+          setRerankSettingModalOpen(true)
         }}
+        disabled={disabled}
       >
-        <Settings04 className="w-[14px] h-[14px]" />
-        <div className='text-xs font-medium'>
-          {t('appDebug.datasetConfig.params')}
-        </div>
-      </div>
+        <RiEqualizer2Line className='mr-1 w-3.5 h-3.5' />
+        {t('dataset.retrievalSettings')}
+      </Button>
       {
-        open && (
+        rerankSettingModalOpen && (
           <Modal
-            isShow={open}
+            isShow={rerankSettingModalOpen}
             onClose={() => {
-              setOpen(false)
+              setRerankSettingModalOpen(false)
             }}
             className='sm:min-w-[528px]'
-            title={t('appDebug.datasetConfig.settingTitle')}
           >
             <ConfigContent
               datasetConfigs={tempDataSetConfigs}
-              onChange={setTempDataSetConfigs}
+              onChange={handleSetTempDataSetConfigs}
+              selectedDatasets={selectedDatasets}
             />
 
             <div className='mt-6 flex justify-end'>
               <Button className='mr-2 flex-shrink-0' onClick={() => {
-                setOpen(false)
+                setRerankSettingModalOpen(false)
               }}>{t('common.operation.cancel')}</Button>
               <Button variant='primary' className='flex-shrink-0' onClick={handleSave} >{t('common.operation.save')}</Button>
             </div>

web/app/components/app/configuration/dataset-config/params-config/weighted-score.tsx

@@ -0,0 +1,112 @@
+import { memo, useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  DEFAULT_WEIGHTED_SCORE,
+  WeightedScoreEnum,
+} from '@/models/datasets'
+import Slider from '@/app/components/base/slider'
+import cn from '@/utils/classnames'
+
+const formatNumber = (value: number) => {
+  if (value > 0 && value < 1)
+    return `0.${value * 10}`
+  else if (value === 1)
+    return '1.0'
+
+  return value
+}
+
+type Value = {
+  type: WeightedScoreEnum
+  value: number[]
+}
+
+type WeightedScoreProps = {
+  value: Value
+  onChange: (value: Value) => void
+}
+const WeightedScore = ({
+  value,
+  onChange = () => {},
+}: WeightedScoreProps) => {
+  const { t } = useTranslation()
+  const options = [
+    {
+      value: WeightedScoreEnum.SemanticFirst,
+      label: t('dataset.weightedScore.semanticFirst'),
+    },
+    {
+      value: WeightedScoreEnum.KeywordFirst,
+      label: t('dataset.weightedScore.keywordFirst'),
+    },
+    {
+      value: WeightedScoreEnum.Customized,
+      label: t('dataset.weightedScore.customized'),
+    },
+  ]
+
+  const disabled = value.type !== WeightedScoreEnum.Customized
+
+  const handleTypeChange = useCallback((type: WeightedScoreEnum) => {
+    const result = { ...value, type }
+
+    if (type === WeightedScoreEnum.SemanticFirst)
+      result.value = [DEFAULT_WEIGHTED_SCORE.semanticFirst.semantic, DEFAULT_WEIGHTED_SCORE.semanticFirst.keyword]
+
+    if (type === WeightedScoreEnum.KeywordFirst)
+      result.value = [DEFAULT_WEIGHTED_SCORE.keywordFirst.semantic, DEFAULT_WEIGHTED_SCORE.keywordFirst.keyword]
+
+    onChange(result)
+  }, [value, onChange])
+
+  return (
+    <div>
+      <div className='flex items-center mb-1 space-x-4'>
+        {
+          options.map(option => (
+            <div
+              key={option.value}
+              className='flex py-1.5 max-w-[calc((100%-32px)/3)] system-sm-regular text-text-secondary cursor-pointer'
+              onClick={() => handleTypeChange(option.value)}
+            >
+              <div
+                className={cn(
+                  'shrink-0 mr-2 w-4 h-4 bg-components-radio-bg border border-components-radio-border rounded-full shadow-xs',
+                  value.type === option.value && 'border-[5px] border-components-radio-border-checked',
+                )}
+              ></div>
+              <div className='truncate' title={option.label}>{option.label}</div>
+            </div>
+          ))
+        }
+      </div>
+      <div className='flex items-center px-3 h-9 space-x-3 rounded-lg border border-components-panel-border'>
+        <div className='shrink-0 flex items-center w-[90px] system-xs-semibold-uppercase text-util-colors-blue-blue-500'>
+          <div className='mr-1 truncate uppercase' title={t('dataset.weightedScore.semantic') || ''}>
+            {t('dataset.weightedScore.semantic')}
+          </div>
+          {formatNumber(value.value[0])}
+        </div>
+        <Slider
+          className={cn('grow h-0.5 bg-gradient-to-r from-[#53B1FD] to-[#2ED3B7]', disabled && 'cursor-not-allowed')}
+          max={1.0}
+          min={0}
+          step={0.1}
+          value={value.value[0]}
+          onChange={v => onChange({ type: value.type, value: [v, (10 - v * 10) / 10] })}
+          disabled={disabled}
+          thumbClassName={cn(disabled && '!cursor-not-allowed')}
+          trackClassName='!bg-transparent'
+        />
+        <div className='shrink-0 flex items-center justify-end w-[90px] system-xs-semibold-uppercase text-util-colors-cyan-cyan-500'>
+          {formatNumber(value.value[1])}
+          <div className='ml-1 truncate uppercase' title={t('dataset.weightedScore.keyword') || ''}>
+            {t('dataset.weightedScore.keyword')}
+          </div>
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default memo(WeightedScore)

web/app/components/app/configuration/dataset-config/select-dataset/index.tsx

@@ -13,7 +13,8 @@ import type { DataSet } from '@/models/datasets'
 import Button from '@/app/components/base/button'
 import { fetchDatasets } from '@/service/datasets'
 import Loading from '@/app/components/base/loading'
-import { formatNumber } from '@/utils/format'
+import Badge from '@/app/components/base/badge'
+import { useKnowledge } from '@/hooks/use-knowledge'
 
 export type ISelectDataSetProps = {
   isShow: boolean
@@ -38,6 +39,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
   const listRef = useRef<HTMLDivElement>(null)
   const [page, setPage, getPage] = useGetState(1)
   const [isNoMore, setIsNoMore] = useState(false)
+  const { formatIndexingTechniqueAndMethod } = useKnowledge()
 
   useInfiniteScroll(
     async () => {
@@ -45,7 +47,7 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
         const { data, has_more } = await fetchDatasets({ url: '/datasets', params: { page } })
         setPage(getPage() + 1)
         setIsNoMore(!has_more)
-        const newList = [...(datasets || []), ...data]
+        const newList = [...(datasets || []), ...data.filter(item => item.indexing_technique)]
         setDataSets(newList)
         setLoaded(true)
         if (!selected.find(item => !item.name))
@@ -136,14 +138,13 @@ const SelectDataSet: FC<ISelectDataSetProps> = ({
                     <span className='ml-1 shrink-0 px-1 border boder-gray-200 rounded-md text-gray-500 text-xs font-normal leading-[18px]'>{t('dataset.unavailable')}</span>
                   )}
                 </div>
-
-                <div className={cn('shrink-0 flex text-xs text-gray-500 overflow-hidden whitespace-nowrap', !item.embedding_available && 'opacity-50')}>
-                  <span className='max-w-[100px] overflow-hidden text-ellipsis whitespace-nowrap'>{formatNumber(item.word_count)}</span>
-                  {t('appDebug.feature.dataSet.words')}
-                  <span className='px-0.5'>·</span>
-                  <span className='max-w-[100px] min-w-[8px] overflow-hidden text-ellipsis whitespace-nowrap'>{formatNumber(item.document_count)} </span>
-                  {t('appDebug.feature.dataSet.textBlocks')}
-                </div>
+                {
+                  item.indexing_technique && (
+                    <Badge
+                      text={formatIndexingTechniqueAndMethod(item.indexing_technique, item.retrieval_model_dict?.search_method)}
+                    />
+                  )
+                }
               </div>
             ))}
           </div>

web/app/components/app/configuration/dataset-config/settings-modal/index.tsx

@@ -259,7 +259,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
 
         {/* Retrieval Method Config */}
         <div className={rowClass}>
-          <div className={labelClass}>
+          <div className={cn(labelClass, 'w-auto min-w-[168px]')}>
             <div>
               <div>{t('datasetSettings.form.retrievalSetting.title')}</div>
               <div className='leading-[18px] text-xs font-normal text-gray-500'>
@@ -268,7 +268,7 @@ const SettingsModal: FC<SettingsModalProps> = ({
               </div>
             </div>
           </div>
-          <div className='w-[480px]'>
+          <div>
             {indexMethod === 'high_quality'
               ? (
                 <RetrievalMethodConfig

web/app/components/app/configuration/index.tsx

@@ -46,7 +46,7 @@ import { fetchDatasets } from '@/service/datasets'
 import { useProviderContext } from '@/context/provider-context'
 import { AgentStrategy, AppType, ModelModeType, RETRIEVE_TYPE, Resolution, TransferMethod } from '@/types/app'
 import { PromptMode } from '@/models/debug'
-import { ANNOTATION_DEFAULT, DEFAULT_AGENT_SETTING, DEFAULT_CHAT_PROMPT_CONFIG, DEFAULT_COMPLETION_PROMPT_CONFIG } from '@/config'
+import { ANNOTATION_DEFAULT, DATASET_DEFAULT, DEFAULT_AGENT_SETTING, DEFAULT_CHAT_PROMPT_CONFIG, DEFAULT_COMPLETION_PROMPT_CONFIG } from '@/config'
 import SelectDataSet from '@/app/components/app/configuration/dataset-config/select-dataset'
 import { useModalContext } from '@/context/modal-context'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
@@ -57,6 +57,10 @@ import { useTextGenerationCurrentProviderAndModelAndModelList } from '@/app/comp
 import { fetchCollectionList } from '@/service/tools'
 import { type Collection } from '@/app/components/tools/types'
 import { useStore as useAppStore } from '@/app/components/app/store'
+import {
+  getMultipleRetrievalConfig,
+  getSelectedDatasetsMode,
+} from '@/app/components/workflow/nodes/knowledge-retrieval/utils'
 
 type PublishConfig = {
   modelConfig: ModelConfig
@@ -174,14 +178,14 @@ const Configuration: FC = () => {
 
   }, [])
   const [datasetConfigs, setDatasetConfigs] = useState<DatasetConfigs>({
-    retrieval_model: RETRIEVE_TYPE.oneWay,
+    retrieval_model: RETRIEVE_TYPE.multiWay,
     reranking_model: {
       reranking_provider_name: '',
       reranking_model_name: '',
     },
-    top_k: 2,
+    top_k: DATASET_DEFAULT.top_k,
     score_threshold_enabled: false,
-    score_threshold: 0.7,
+    score_threshold: DATASET_DEFAULT.score_threshold,
     datasets: {
       datasets: [],
     },
@@ -202,6 +206,7 @@ const Configuration: FC = () => {
   const hasSetContextVar = !!contextVar
   const [isShowSelectDataSet, { setTrue: showSelectDataSet, setFalse: hideSelectDataSet }] = useBoolean(false)
   const selectedIds = dataSets.map(item => item.id)
+  const [rerankSettingModalOpen, setRerankSettingModalOpen] = useState(false)
   const handleSelect = (data: DataSet[]) => {
     if (isEqual(data.map(item => item.id), dataSets.map(item => item.id))) {
       hideSelectDataSet()
@@ -209,6 +214,7 @@ const Configuration: FC = () => {
     }
 
     formattingChangedDispatcher()
+    let newDatasets = data
     if (data.find(item => !item.name)) { // has not loaded selected dataset
       const newSelected = produce(data, (draft: any) => {
         data.forEach((item, index) => {
@@ -220,11 +226,45 @@ const Configuration: FC = () => {
         })
       })
       setDataSets(newSelected)
+      newDatasets = newSelected
     }
     else {
       setDataSets(data)
     }
     hideSelectDataSet()
+    const {
+      allEconomic,
+      mixtureHighQualityAndEconomic,
+      inconsistentEmbeddingModel,
+    } = getSelectedDatasetsMode(newDatasets)
+
+    if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel)
+      setRerankSettingModalOpen(true)
+
+    const { datasets, retrieval_model, score_threshold_enabled, ...restConfigs } = datasetConfigs
+
+    const retrievalConfig = getMultipleRetrievalConfig({
+      top_k: restConfigs.top_k,
+      score_threshold: restConfigs.score_threshold,
+      reranking_model: restConfigs.reranking_model && {
+        provider: restConfigs.reranking_model.reranking_provider_name,
+        model: restConfigs.reranking_model.reranking_model_name,
+      },
+      reranking_mode: restConfigs.reranking_mode,
+      weights: restConfigs.weights,
+      reranking_enable: restConfigs.reranking_enable,
+    }, newDatasets)
+
+    setDatasetConfigs({
+      ...retrievalConfig,
+      reranking_model: restConfigs.reranking_model && {
+        reranking_provider_name: restConfigs.reranking_model.reranking_provider_name,
+        reranking_model_name: restConfigs.reranking_model.reranking_model_name,
+      },
+      retrieval_model,
+      score_threshold_enabled,
+      datasets,
+    })
   }
 
   const [isShowHistoryModal, { setTrue: showHistoryModal, setFalse: hideHistoryModal }] = useBoolean(false)
@@ -509,7 +549,7 @@ const Configuration: FC = () => {
         syncToPublishedConfig(config)
         setPublishedConfig(config)
         setDatasetConfigs({
-          retrieval_model: RETRIEVE_TYPE.oneWay,
+          retrieval_model: RETRIEVE_TYPE.multiWay,
           ...modelConfig.dataset_configs,
         })
         setHasFetchedDetail(true)
@@ -744,6 +784,8 @@ const Configuration: FC = () => {
       isShowVisionConfig,
       visionConfig,
       setVisionConfig: handleSetVisionConfig,
+      rerankSettingModalOpen,
+      setRerankSettingModalOpen,
     }}
     >
       <>

web/app/components/base/badge.tsx

@@ -0,0 +1,25 @@
+import { memo } from 'react'
+import cn from '@/utils/classnames'
+
+type BadgeProps = {
+  className?: string
+  text: string
+}
+
+const Badge = ({
+  className,
+  text,
+}: BadgeProps) => {
+  return (
+    <div
+      className={cn(
+        'inline-flex items-center px-[5px] h-5 rounded-[5px] border border-divider-deep system-2xs-medium-uppercase leading-3 text-text-tertiary',
+        className,
+      )}
+    >
+      {text}
+    </div>
+  )
+}
+
+export default memo(Badge)

web/app/components/base/button/index.css

@@ -2,46 +2,185 @@
 
 @layer components {
   .btn {
-    @apply inline-flex justify-center items-center border-[0.5px] font-medium cursor-pointer whitespace-nowrap shadow;
+    @apply inline-flex justify-center items-center cursor-pointer whitespace-nowrap;
   }
 
   .btn-disabled {
-    @apply opacity-60 cursor-not-allowed;
+    @apply cursor-not-allowed;
   }
 
   .btn-small {
-    @apply px-2 h-6 rounded-md text-xs
+    @apply px-2 h-6 rounded-md text-xs font-medium;
   }
 
   .btn-medium {
-    @apply px-3.5 h-8 rounded-lg text-[13px]
+    @apply px-3.5 h-8 rounded-lg text-[13px] leading-4 font-medium;
   }
 
   .btn-large {
-    @apply px-4 h-9 rounded-[10px] text-sm font-semibold
-  }
-
-  .btn-secondary {
-    @apply bg-white hover:bg-white/80 border-gray-200 hover:border-gray-300 text-gray-700;
-  }
-
-  .btn-secondary-accent {
-    @apply bg-white hover:bg-white/80 border-gray-200 hover:border-gray-300 text-primary-600;
+    @apply px-4 h-9 rounded-[10px] text-sm font-semibold;
   }
 
   .btn-primary {
-    @apply bg-primary-600 hover:bg-primary-700 text-white;
+    @apply 
+    shadow
+    bg-components-button-primary-bg
+    border-components-button-primary-border
+    hover:bg-components-button-primary-bg-hover
+    hover:border-components-button-primary-border-hover
+    text-components-button-primary-text;
+  }
+
+  .btn-primary.btn-destructive {
+    @apply 
+    bg-components-button-destructive-primary-bg
+    border-components-button-destructive-primary-border
+    hover:bg-components-button-destructive-primary-bg-hover
+    hover:border-components-button-destructive-primary-border-hover
+    text-components-button-destructive-primary-text;
+  }
+
+  .btn-primary.btn-disabled {
+    @apply 
+    shadow-none
+    bg-components-button-primary-bg-disabled
+    border-components-button-primary-border-disabled
+    text-components-button-primary-text-disabled;
+  }
+
+  .btn-primary.btn-destructive.btn-disabled {
+    @apply 
+    shadow-none
+    bg-components-button-destructive-primary-bg-disabled
+    border-components-button-destructive-primary-border-disabled
+    text-components-button-destructive-primary-text-disabled;
+  }
+
+  .btn-secondary {
+    @apply 
+    border-[0.5px]
+    shadow-xs
+    bg-components-button-secondary-bg 
+    border-components-button-secondary-border 
+    hover:bg-components-button-secondary-bg-hover 
+    hover:border-components-button-secondary-border-hover 
+    text-components-button-secondary-text;
+  }
+
+  .btn-secondary.btn-disabled {
+    @apply 
+    bg-components-button-secondary-bg-disabled 
+    border-components-button-secondary-border-disabled 
+    text-components-button-secondary-text-disabled;
+  }
+
+  .btn-secondary.btn-destructive {
+    @apply 
+    bg-components-button-destructive-secondary-bg 
+    border-components-button-destructive-secondary-border 
+    hover:bg-components-button-destructive-secondary-bg-hover 
+    hover:border-components-button-destructive-secondary-border-hover 
+    text-components-button-destructive-secondary-text;
+  }
+
+  .btn-secondary.btn-destructive.btn-disabled {
+    @apply 
+    bg-components-button-destructive-secondary-bg-disabled 
+    border-components-button-destructive-secondary-border-disabled 
+    text-components-button-destructive-secondary-text-disabled;
+  }
+
+  
+  .btn-secondary-accent {
+    @apply 
+    border-[0.5px]
+    shadow-xs
+    bg-components-button-secondary-bg 
+    border-components-button-secondary-border 
+    hover:bg-components-button-secondary-bg-hover 
+    hover:border-components-button-secondary-border-hover 
+    text-components-button-secondary-accent-text;
+  }
+
+  .btn-secondary-accent.btn-disabled {
+    @apply 
+    bg-components-button-secondary-bg-disabled 
+    border-components-button-secondary-border-disabled 
+    text-components-button-secondary-accent-text-disabled;
   }
 
   .btn-warning {
-    @apply bg-red-600 hover:bg-red-700 text-white;
+    @apply 
+    bg-components-button-destructive-primary-bg 
+    border-components-button-destructive-primary-border 
+    hover:bg-components-button-destructive-primary-bg-hover 
+    hover:border-components-button-destructive-primary-border-hover 
+    text-components-button-destructive-primary-text;
   }
 
-  .btn-ghost {
-    @apply bg-transparent hover:bg-gray-200 border-transparent shadow-none text-gray-700;
+  .btn-warning.btn-disabled {
+    @apply 
+    bg-components-button-destructive-primary-bg-disabled 
+    border-components-button-destructive-primary-border-disabled 
+    text-components-button-destructive-primary-text-disabled;
   }
 
   .btn-tertiary {
-    @apply bg-[#F2F4F7] hover:bg-[#E9EBF0] border-transparent shadow-none text-gray-700;
+    @apply 
+    bg-components-button-tertiary-bg 
+    hover:bg-components-button-tertiary-bg-hover 
+    text-components-button-tertiary-text;
+  }
+
+  .btn-tertiary.btn-disabled {
+    @apply 
+    bg-components-button-tertiary-bg-disabled 
+    text-components-button-tertiary-text-disabled;
+  }
+
+  .btn-tertiary.btn-destructive {
+    @apply 
+    bg-components-button-destructive-tertiary-bg 
+    hover:bg-components-button-destructive-tertiary-bg-hover 
+    text-components-button-destructive-tertiary-text;
+  }
+
+  .btn-tertiary.btn-destructive.btn-disabled {
+    @apply 
+    bg-components-button-destructive-tertiary-bg-disabled 
+    text-components-button-destructive-tertiary-text-disabled;
+  }
+
+  .btn-ghost {
+    @apply 
+    hover:bg-components-button-ghost-bg-hover 
+    text-components-button-ghost-text;
+  }
+
+  .btn-ghost.btn-disabled {
+    @apply 
+    text-components-button-ghost-text-disabled;
+  }
+
+  .btn-ghost.btn-destructive {
+    @apply 
+    hover:bg-components-button-destructive-ghost-bg-hover 
+    text-components-button-destructive-ghost-text;
+  }
+
+  .btn-ghost.btn-destructive.btn-disabled {
+    @apply 
+    text-components-button-destructive-ghost-text-disabled;
+  }
+
+  .btn-ghost-accent {
+    @apply 
+    hover:bg-state-accent-hover
+    text-components-button-secondary-accent-text;
+  }
+
+  .btn-ghost-accent.btn-disabled {
+    @apply 
+    text-components-button-secondary-accent-text-disabled;
   }
 }

web/app/components/base/button/index.tsx

@@ -14,6 +14,7 @@ const buttonVariants = cva(
         'secondary': 'btn-secondary',
         'secondary-accent': 'btn-secondary-accent',
         'ghost': 'btn-ghost',
+        'ghost-accent': 'btn-ghost-accent',
         'tertiary': 'btn-tertiary',
       },
       size: {
@@ -30,16 +31,20 @@ const buttonVariants = cva(
 )
 
 export type ButtonProps = {
+  destructive?: boolean
   loading?: boolean
   styleCss?: CSSProperties
 } & React.ButtonHTMLAttributes<HTMLButtonElement> & VariantProps<typeof buttonVariants>
 
 const Button = React.forwardRef<HTMLButtonElement, ButtonProps>(
-  ({ className, variant, size, loading, styleCss, children, ...props }, ref) => {
+  ({ className, variant, size, destructive, loading, styleCss, children, ...props }, ref) => {
     return (
       <button
         type='button'
-        className={classNames(buttonVariants({ variant, size, className }))}
+        className={classNames(
+          buttonVariants({ variant, size, className }),
+          destructive && 'btn-destructive',
+        )}
         ref={ref}
         style={styleCss}
         {...props}

web/app/components/base/chat/embedded-chatbot/chat-wrapper.tsx

@@ -111,7 +111,7 @@ const ChatWrapper = () => {
       )
     }
 
-    return <div className='mb-6' />
+    return null
   }, [currentConversationId, inputsForms, isMobile])
 
   return (

web/app/components/base/chat/embedded-chatbot/index.tsx

@@ -3,6 +3,8 @@ import {
   useState,
 } from 'react'
 import { useAsyncEffect } from 'ahooks'
+import { useTranslation } from 'react-i18next'
+import { RiLoopLeftLine } from '@remixicon/react'
 import {
   EmbeddedChatbotContext,
   useEmbeddedChatbotContext,
@@ -19,8 +21,10 @@ import LogoHeader from '@/app/components/base/logo/logo-embeded-chat-header'
 import Header from '@/app/components/base/chat/embedded-chatbot/header'
 import ConfigPanel from '@/app/components/base/chat/embedded-chatbot/config-panel'
 import ChatWrapper from '@/app/components/base/chat/embedded-chatbot/chat-wrapper'
+import Tooltip from '@/app/components/base/tooltip'
 
 const Chatbot = () => {
+  const { t } = useTranslation()
   const {
     isMobile,
     appInfoError,
@@ -80,7 +84,22 @@ const Chatbot = () => {
             <Loading type='app' />
           )}
           {chatReady && !appChatListDataLoading && (
-            <ChatWrapper />
+            <div className='relative h-full pt-8 mx-auto w-full max-w-[720px]'>
+              {!isMobile && (
+                <div className='absolute top-2.5 right-3 z-20'>
+                  <Tooltip
+                    selector={'embed-scene-restart-button'}
+                    htmlContent={t('share.chat.resetChat')}
+                    position='top'
+                  >
+                    <div className='p-1.5 bg-white border-[0.5px] border-gray-100 rounded-lg shadow-md cursor-pointer' onClick={handleNewConversation}>
+                      <RiLoopLeftLine className="h-4 w-4 text-gray-500"/>
+                    </div>
+                  </Tooltip>
+                </div>
+              )}
+              <ChatWrapper />
+            </div>
           )}
         </div>
       </div>

web/app/components/base/radio-card/index.tsx

@@ -1,7 +1,6 @@
 'use client'
 import type { FC } from 'react'
 import React from 'react'
-import s from './style.module.css'
 import cn from '@/utils/classnames'
 
 type Props = {
@@ -29,7 +28,12 @@ const RadioCard: FC<Props> = ({
   chosenConfigWrapClassName,
 }) => {
   return (
-    <div className={cn(s.item, isChosen && s.active)}>
+    <div
+      className={cn(
+        'border border-components-option-card-option-border bg-components-option-card-option-bg rounded-xl hover:shadow-xs cursor-pointer',
+        isChosen && 'bg-components-option-card-option-selected-bg border-components-panel-border shadow-xs',
+      )}
+    >
       <div className='flex py-3 pl-3 pr-4' onClick={onChosen}>
         <div className={cn(iconBgClassName, 'mr-3 shrink-0 flex w-8 justify-center h-8 items-center rounded-lg')}>
           {icon}
@@ -40,12 +44,15 @@ const RadioCard: FC<Props> = ({
         </div>
         {!noRadio && (
           <div className='shrink-0 flex items-center h-8'>
-            <div className={s.radio}></div>
+            <div className={cn(
+              'w-4 h-4 border border-components-radio-border bg-components-radio-bg shadow-xs rounded-full',
+              isChosen && 'border-[5px] border-components-radio-border-checked',
+            )}></div>
           </div>
         )}
       </div>
       {((isChosen && chosenConfig) || noRadio) && (
-        <div className={cn(chosenConfigWrapClassName, 'pt-2 px-14 pb-6 border-t border-gray-200')}>
+        <div className={cn(chosenConfigWrapClassName, 'p-3 border-t border-gray-200')}>
           {chosenConfig}
         </div>
       )}

web/app/components/base/radio-card/simple/index.tsx

@@ -6,7 +6,7 @@ import cn from '@/utils/classnames'
 
 type Props = {
   className?: string
-  title: string
+  title: string | JSX.Element | null
   description: string
   isChosen: boolean
   onChosen: () => void

web/app/components/base/radio-card/style.module.css

@@ -1,25 +0,0 @@
-.item {
-  @apply relative rounded-xl border border-gray-100 cursor-pointer;
-  background-color: #fcfcfd;
-}
-
-.item.active {
-  border-width: 1.5px;
-  border-color: #528BFF;
-  box-shadow: 0px 1px 3px rgba(16, 24, 40, 0.1), 0px 1px 2px rgba(16, 24, 40, 0.06);
-}
-
-.item:hover {
-  background-color: #ffffff;
-  border-color: #B2CCFF;
-  box-shadow: 0px 12px 16px -4px rgba(16, 24, 40, 0.08), 0px 4px 6px -2px rgba(16, 24, 40, 0.03);
-}
-
-.radio {
-  @apply w-4 h-4 border-[2px] border-gray-200 rounded-full;
-}
-
-.item.active .radio {
-  border-width: 5px;
-  border-color: #155EEF;
-}

web/app/components/base/slider/index.tsx

@@ -4,6 +4,8 @@ import './style.css'
 
 type ISliderProps = {
   className?: string
+  thumbClassName?: string
+  trackClassName?: string
   value: number
   max?: number
   min?: number
@@ -12,16 +14,26 @@ type ISliderProps = {
   onChange: (value: number) => void
 }
 
-const Slider: React.FC<ISliderProps> = ({ className, max, min, step, value, disabled, onChange }) => {
+const Slider: React.FC<ISliderProps> = ({
+  className,
+  thumbClassName,
+  trackClassName,
+  max,
+  min,
+  step,
+  value,
+  disabled,
+  onChange,
+}) => {
   return <ReactSlider
     disabled={disabled}
     value={isNaN(value) ? 0 : value}
     min={min || 0}
     max={max || 100}
     step={step || 1}
-    className={cn(className, 'slider')}
-    thumbClassName="slider-thumb"
-    trackClassName="slider-track"
+    className={cn('slider', className)}
+    thumbClassName={cn('slider-thumb', thumbClassName)}
+    trackClassName={cn('slider-track', trackClassName)}
     onChange={onChange}
   />
 }

web/app/components/datasets/common/check-rerank-model.ts

@@ -3,6 +3,7 @@ import type {
   DefaultModelResponse,
   Model,
 } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import { RerankingModeEnum } from '@/models/datasets'
 
 export const isReRankModelSelected = ({
   rerankDefaultModel,
@@ -32,7 +33,7 @@ export const isReRankModelSelected = ({
 
   if (
     indexMethod === 'high_quality'
-    && (retrievalConfig.reranking_enable || retrievalConfig.search_method === RETRIEVE_METHOD.hybrid)
+    && (retrievalConfig.search_method === RETRIEVE_METHOD.hybrid && retrievalConfig.reranking_mode !== RerankingModeEnum.WeightedScore)
     && !rerankModelSelected
   )
     return false

web/app/components/datasets/common/retrieval-param-config/index.tsx

@@ -15,6 +15,11 @@ import type { RetrievalConfig } from '@/types/app'
 import ModelSelector from '@/app/components/header/account-setting/model-provider-page/model-selector'
 import { useModelListAndDefaultModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
+import {
+  RerankingModeEnum,
+  WeightedScoreEnum,
+} from '@/models/datasets'
+import WeightedScore from '@/app/components/app/configuration/dataset-config/params-config/weighted-score'
 
 type Props = {
   type: RETRIEVE_METHOD
@@ -34,6 +39,7 @@ const RetrievalParamConfig: FC<Props> = ({
     defaultModel: rerankDefaultModel,
     modelList: rerankModelList,
   } = useModelListAndDefaultModel(ModelTypeEnum.rerank)
+  const isHybridSearch = type === RETRIEVE_METHOD.hybrid
 
   const rerankModel = (() => {
     if (value.reranking_model) {
@@ -50,9 +56,47 @@ const RetrievalParamConfig: FC<Props> = ({
     }
   })()
 
+  const handleChangeRerankMode = (v: RerankingModeEnum) => {
+    if (v === value.reranking_mode)
+      return
+
+    const result = {
+      ...value,
+      reranking_mode: v,
+    }
+
+    if (!result.weights && v === RerankingModeEnum.WeightedScore) {
+      result.weights = {
+        weight_type: WeightedScoreEnum.Customized,
+        vector_setting: {
+          vector_weight: 0.5,
+          embedding_provider_name: '',
+          embedding_model_name: '',
+        },
+        keyword_setting: {
+          keyword_weight: 0.5,
+        },
+      }
+    }
+    onChange(result)
+  }
+
+  const rerankingModeOptions = [
+    {
+      value: RerankingModeEnum.WeightedScore,
+      label: t('dataset.weightedScore.title'),
+      tips: t('dataset.weightedScore.description'),
+    },
+    {
+      value: RerankingModeEnum.RerankingModel,
+      label: t('common.modelProvider.rerankModel.key'),
+      tips: t('common.modelProvider.rerankModel.tip'),
+    },
+  ]
+
   return (
     <div>
-      {!isEconomical && (
+      {!isEconomical && !isHybridSearch && (
         <div>
           <div className='flex h-8 items-center text-[13px] font-medium text-gray-900 space-x-2'>
             {canToggleRerankModalEnable && (
@@ -75,10 +119,10 @@ const RetrievalParamConfig: FC<Props> = ({
             </div>
           </div>
           <ModelSelector
-            triggerClassName={`${!value.reranking_enable && type !== RETRIEVE_METHOD.hybrid && '!opacity-60 !cursor-not-allowed'}`}
+            triggerClassName={`${!value.reranking_enable && '!opacity-60 !cursor-not-allowed'}`}
             defaultModel={rerankModel && { provider: rerankModel.provider_name, model: rerankModel.model_name }}
             modelList={rerankModelList}
-            readonly={!value.reranking_enable && type !== RETRIEVE_METHOD.hybrid}
+            readonly={!value.reranking_enable}
             onSelect={(v) => {
               onChange({
                 ...value,
@@ -91,40 +135,152 @@ const RetrievalParamConfig: FC<Props> = ({
           />
         </div>
       )}
-
-      <div className={cn(!isEconomical && 'mt-4', 'flex space-between space-x-6')}>
-        <TopKItem
-          className='grow'
-          value={value.top_k}
-          onChange={(_key, v) => {
-            onChange({
-              ...value,
-              top_k: v,
-            })
-          }}
-          enable={true}
-        />
-        {(!isEconomical && !(value.search_method === RETRIEVE_METHOD.fullText && !value.reranking_enable)) && (
-          <ScoreThresholdItem
-            className='grow'
-            value={value.score_threshold}
-            onChange={(_key, v) => {
-              onChange({
-                ...value,
-                score_threshold: v,
-              })
-            }}
-            enable={value.score_threshold_enabled}
-            hasSwitch={true}
-            onSwitchChange={(_key, v) => {
-              onChange({
-                ...value,
-                score_threshold_enabled: v,
-              })
-            }}
-          />
-        )}
-      </div>
+      {
+        !isHybridSearch && (
+          <div className={cn(!isEconomical && 'mt-4', 'flex space-between space-x-6')}>
+            <TopKItem
+              className='grow'
+              value={value.top_k}
+              onChange={(_key, v) => {
+                onChange({
+                  ...value,
+                  top_k: v,
+                })
+              }}
+              enable={true}
+            />
+            {(!isEconomical && !(value.search_method === RETRIEVE_METHOD.fullText && !value.reranking_enable)) && (
+              <ScoreThresholdItem
+                className='grow'
+                value={value.score_threshold}
+                onChange={(_key, v) => {
+                  onChange({
+                    ...value,
+                    score_threshold: v,
+                  })
+                }}
+                enable={value.score_threshold_enabled}
+                hasSwitch={true}
+                onSwitchChange={(_key, v) => {
+                  onChange({
+                    ...value,
+                    score_threshold_enabled: v,
+                  })
+                }}
+              />
+            )}
+          </div>
+        )
+      }
+      {
+        isHybridSearch && (
+          <>
+            <div className='flex items-center justify-between'>
+              {
+                rerankingModeOptions.map(option => (
+                  <div
+                    key={option.value}
+                    className={cn(
+                      'flex items-center justify-center mb-4 w-[calc((100%-8px)/2)] h-8 rounded-lg border border-components-option-card-option-border bg-components-option-card-option-bg cursor-pointer system-sm-medium text-text-secondary',
+                      value.reranking_mode === RerankingModeEnum.WeightedScore && option.value === RerankingModeEnum.WeightedScore && 'border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary',
+                      value.reranking_mode !== RerankingModeEnum.WeightedScore && option.value !== RerankingModeEnum.WeightedScore && 'border-[1.5px] border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg text-text-primary',
+                    )}
+                    onClick={() => handleChangeRerankMode(option.value)}
+                  >
+                    <div className='truncate'>{option.label}</div>
+                    <Tooltip
+                      popupContent={<div className='w-[200px]'>{option.tips}</div>}
+                      hideArrow
+                    >
+                      <RiQuestionLine className='ml-0.5 w-3.5 h-4.5 text-text-quaternary' />
+                    </Tooltip>
+                  </div>
+                ))
+              }
+            </div>
+            {
+              value.reranking_mode === RerankingModeEnum.WeightedScore && (
+                <WeightedScore
+                  value={{
+                    type: value.weights!.weight_type,
+                    value: [
+                      value.weights!.vector_setting.vector_weight,
+                      value.weights!.keyword_setting.keyword_weight,
+                    ],
+                  }}
+                  onChange={(v) => {
+                    onChange({
+                      ...value,
+                      weights: {
+                        ...value.weights!,
+                        weight_type: v.type,
+                        vector_setting: {
+                          ...value.weights!.vector_setting,
+                          vector_weight: v.value[0],
+                        },
+                        keyword_setting: {
+                          ...value.weights!.keyword_setting,
+                          keyword_weight: v.value[1],
+                        },
+                      },
+                    })
+                  }}
+                />
+              )
+            }
+            {
+              value.reranking_mode !== RerankingModeEnum.WeightedScore && (
+                <ModelSelector
+                  triggerClassName={`${!value.reranking_enable && '!opacity-60 !cursor-not-allowed'}`}
+                  defaultModel={rerankModel && { provider: rerankModel.provider_name, model: rerankModel.model_name }}
+                  modelList={rerankModelList}
+                  readonly={!value.reranking_enable}
+                  onSelect={(v) => {
+                    onChange({
+                      ...value,
+                      reranking_model: {
+                        reranking_provider_name: v.provider,
+                        reranking_model_name: v.model,
+                      },
+                    })
+                  }}
+                />
+              )
+            }
+            <div className={cn(!isEconomical && 'mt-4', 'flex space-between space-x-6')}>
+              <TopKItem
+                className='grow'
+                value={value.top_k}
+                onChange={(_key, v) => {
+                  onChange({
+                    ...value,
+                    top_k: v,
+                  })
+                }}
+                enable={true}
+              />
+              <ScoreThresholdItem
+                className='grow'
+                value={value.score_threshold}
+                onChange={(_key, v) => {
+                  onChange({
+                    ...value,
+                    score_threshold: v,
+                  })
+                }}
+                enable={value.score_threshold_enabled}
+                hasSwitch={true}
+                onSwitchChange={(_key, v) => {
+                  onChange({
+                    ...value,
+                    score_threshold_enabled: v,
+                  })
+                }}
+              />
+            </div>
+          </>
+        )
+      }
     </div>
   )
 }

web/app/components/datasets/settings/form/index.tsx

@@ -113,6 +113,10 @@ const Form = () => {
       retrievalConfig,
       indexMethod,
     })
+    if (postRetrievalConfig.weights) {
+      postRetrievalConfig.weights.vector_setting.embedding_provider_name = currentDataset?.embedding_model_provider || ''
+      postRetrievalConfig.weights.vector_setting.embedding_model_name = currentDataset?.embedding_model || ''
+    }
     try {
       setLoading(true)
       const requestParams = {

web/app/components/tools/add-tool-modal/category.tsx

@@ -22,7 +22,6 @@ const Icon = ({ svgString, active }: { svgString: string; active: boolean }) =>
       return null
     const parser = new DOMParser()
     const doc = parser.parseFromString(svg, 'image/svg+xml')
-    console.log(doc.documentElement)
     return doc.documentElement
   }
   useMount(() => {

web/app/components/workflow/index.tsx

@@ -248,6 +248,7 @@ const Workflow: FC<WorkflowProps> = memo(({
   } = useWorkflow()
   const { handleStartWorkflowRun } = useWorkflowStartRun()
   const {
+    exportCheck,
     handleExportDSL,
   } = useDSL()
 
@@ -334,7 +335,7 @@ const Workflow: FC<WorkflowProps> = memo(({
         showImportDSLModal && (
           <UpdateDSLModal
             onCancel={() => setShowImportDSLModal(false)}
-            onBackup={handleExportDSL}
+            onBackup={exportCheck}
             onImport={handlePaneContextmenuCancel}
           />
         )

web/app/components/workflow/nodes/knowledge-retrieval/components/dataset-item.tsx

@@ -4,15 +4,17 @@ import React, { useCallback } from 'react'
 import { useBoolean } from 'ahooks'
 import {
   RiDeleteBinLine,
+  RiEditLine,
 } from '@remixicon/react'
 import type { DataSet } from '@/models/datasets'
 import { DataSourceType } from '@/models/datasets'
-import { Settings01 } from '@/app/components/base/icons/src/vender/line/general'
 import FileIcon from '@/app/components/base/file-icon'
 import { Folder } from '@/app/components/base/icons/src/vender/solid/files'
 import SettingsModal from '@/app/components/app/configuration/dataset-config/settings-modal'
 import Drawer from '@/app/components/base/drawer'
 import useBreakpoints, { MediaType } from '@/hooks/use-breakpoints'
+import Badge from '@/app/components/base/badge'
+import { useKnowledge } from '@/hooks/use-knowledge'
 
 type Props = {
   payload: DataSet
@@ -29,6 +31,7 @@ const DatasetItem: FC<Props> = ({
 }) => {
   const media = useBreakpoints()
   const isMobile = media === MediaType.mobile
+  const { formatIndexingTechniqueAndMethod } = useKnowledge()
 
   const [isShowSettingsModal, {
     setTrue: showSettingsModal,
@@ -62,7 +65,7 @@ const DatasetItem: FC<Props> = ({
             className='flex items-center justify-center w-6 h-6 hover:bg-black/5 rounded-md cursor-pointer'
             onClick={showSettingsModal}
           >
-            <Settings01 className='w-4 h-4 text-gray-500' />
+            <RiEditLine className='w-4 h-4 text-gray-500' />
           </div>
           <div
             className='flex items-center justify-center w-6 h-6 hover:bg-black/5 rounded-md cursor-pointer'
@@ -72,6 +75,10 @@ const DatasetItem: FC<Props> = ({
           </div>
         </div>
       )}
+      <Badge
+        className='group-hover/dataset-item:hidden shrink-0'
+        text={formatIndexingTechniqueAndMethod(payload.indexing_technique, payload.retrieval_model_dict?.search_method)}
+      />
 
       {isShowSettingsModal && (
         <Drawer isOpen={isShowSettingsModal} onClose={hideSettingsModal} footer={null} mask={isMobile} panelClassname='mt-16 mx-2 sm:mr-2 mb-3 !p-0 !max-w-[640px] rounded-xl'>

web/app/components/workflow/nodes/knowledge-retrieval/components/retrieval-config.tsx

@@ -1,8 +1,8 @@
 'use client'
 import type { FC } from 'react'
 import React, { useCallback, useState } from 'react'
+import { RiEqualizer2Line } from '@remixicon/react'
 import { useTranslation } from 'react-i18next'
-import { RiArrowDownSLine } from '@remixicon/react'
 import type { MultipleRetrievalConfig, SingleRetrievalConfig } from '../types'
 import type { ModelConfig } from '../../../types'
 import cn from '@/utils/classnames'
@@ -16,10 +16,9 @@ import { RETRIEVE_TYPE } from '@/types/app'
 import { DATASET_DEFAULT } from '@/config'
 import { useModelListAndDefaultModelAndCurrentProviderAndModel } from '@/app/components/header/account-setting/model-provider-page/hooks'
 import { ModelTypeEnum } from '@/app/components/header/account-setting/model-provider-page/declarations'
-
-import type {
-  DatasetConfigs,
-} from '@/models/debug'
+import Button from '@/app/components/base/button'
+import type { DatasetConfigs } from '@/models/debug'
+import type { DataSet } from '@/models/datasets'
 
 type Props = {
   payload: {
@@ -33,6 +32,9 @@ type Props = {
   onSingleRetrievalModelChange?: (config: ModelConfig) => void
   onSingleRetrievalModelParamsChange?: (config: ModelConfig) => void
   readonly?: boolean
+  openFromProps?: boolean
+  onOpenFromPropsChange?: (openFromProps: boolean) => void
+  selectedDatasets: DataSet[]
 }
 
 const RetrievalConfig: FC<Props> = ({
@@ -43,10 +45,18 @@ const RetrievalConfig: FC<Props> = ({
   onSingleRetrievalModelChange,
   onSingleRetrievalModelParamsChange,
   readonly,
+  openFromProps,
+  onOpenFromPropsChange,
+  selectedDatasets,
 }) => {
   const { t } = useTranslation()
-
   const [open, setOpen] = useState(false)
+  const mergedOpen = openFromProps !== undefined ? openFromProps : open
+
+  const handleOpen = useCallback((newOpen: boolean) => {
+    setOpen(newOpen)
+    onOpenFromPropsChange?.(newOpen)
+  }, [onOpenFromPropsChange])
 
   const {
     defaultModel: rerankDefaultModel,
@@ -72,16 +82,18 @@ const RetrievalConfig: FC<Props> = ({
             provider: configs.reranking_model?.reranking_provider_name,
             model: configs.reranking_model?.reranking_model_name,
           }),
+      reranking_mode: configs.reranking_mode,
+      weights: configs.weights as any,
+      reranking_enable: configs.reranking_enable,
     })
   }, [onMultipleRetrievalConfigChange, payload.retrieval_mode, rerankDefaultModel?.provider?.provider, rerankDefaultModel?.model, onRetrievalModeChange])
 
   return (
     <PortalToFollowElem
-      open={open}
-      onOpenChange={setOpen}
+      open={mergedOpen}
+      onOpenChange={handleOpen}
       placement='bottom-end'
       offset={{
-        // mainAxis: 12,
         crossAxis: -2,
       }}
     >
@@ -89,13 +101,18 @@ const RetrievalConfig: FC<Props> = ({
         onClick={() => {
           if (readonly)
             return
-          setOpen(v => !v)
+          handleOpen(!mergedOpen)
         }}
       >
-        <div className={cn(!readonly && 'cursor-pointer', open && 'bg-gray-100', 'flex items-center h-6  px-2 rounded-md hover:bg-gray-100 group  select-none')}>
-          <div className={cn(open ? 'text-gray-700' : 'text-gray-500', 'leading-[18px] text-xs font-medium group-hover:bg-gray-100')}>{payload.retrieval_mode === RETRIEVE_TYPE.oneWay ? t('appDebug.datasetConfig.retrieveOneWay.title') : t('appDebug.datasetConfig.retrieveMultiWay.title')}</div>
-          {!readonly && <RiArrowDownSLine className='w-3 h-3 ml-1' />}
-        </div>
+        <Button
+          variant='ghost'
+          size='small'
+          disabled={readonly}
+          className={cn(open && 'bg-components-button-ghost-bg-hover')}
+        >
+          <RiEqualizer2Line className='mr-1 w-3.5 h-3.5' />
+          {t('dataset.retrievalSettings')}
+        </Button>
       </PortalToFollowElemTrigger>
       <PortalToFollowElemContent style={{ zIndex: 1001 }}>
         <div className='w-[404px] pt-3 pb-4 px-4 shadow-xl  rounded-2xl border border-gray-200  bg-white'>
@@ -103,21 +120,24 @@ const RetrievalConfig: FC<Props> = ({
             datasetConfigs={
               {
                 retrieval_model: payload.retrieval_mode,
-                reranking_model: !multiple_retrieval_config?.reranking_model?.provider
+                reranking_model: multiple_retrieval_config?.reranking_model?.provider
                   ? {
-                    reranking_provider_name: rerankDefaultModel?.provider?.provider || '',
-                    reranking_model_name: rerankDefaultModel?.model || '',
+                    reranking_provider_name: multiple_retrieval_config.reranking_model?.provider,
+                    reranking_model_name: multiple_retrieval_config.reranking_model?.model,
                   }
                   : {
-                    reranking_provider_name: multiple_retrieval_config?.reranking_model?.provider || '',
-                    reranking_model_name: multiple_retrieval_config?.reranking_model?.model || '',
+                    reranking_provider_name: '',
+                    reranking_model_name: '',
                   },
                 top_k: multiple_retrieval_config?.top_k || DATASET_DEFAULT.top_k,
-                score_threshold_enabled: !(multiple_retrieval_config?.score_threshold === undefined || multiple_retrieval_config?.score_threshold === null),
+                score_threshold_enabled: !(multiple_retrieval_config?.score_threshold === undefined || multiple_retrieval_config.score_threshold === null),
                 score_threshold: multiple_retrieval_config?.score_threshold,
                 datasets: {
                   datasets: [],
                 },
+                reranking_mode: multiple_retrieval_config?.reranking_mode,
+                weights: multiple_retrieval_config?.weights,
+                reranking_enable: multiple_retrieval_config?.reranking_enable,
               }
             }
             onChange={handleChange}
@@ -125,6 +145,7 @@ const RetrievalConfig: FC<Props> = ({
             singleRetrievalModelConfig={singleRetrievalModelConfig}
             onSingleRetrievalModelChange={onSingleRetrievalModelChange}
             onSingleRetrievalModelParamsChange={onSingleRetrievalModelParamsChange}
+            selectedDatasets={selectedDatasets}
           />
         </div>
       </PortalToFollowElemContent>

web/app/components/workflow/nodes/knowledge-retrieval/default.ts

@@ -2,7 +2,7 @@ import { BlockEnum } from '../../types'
 import type { NodeDefault } from '../../types'
 import type { KnowledgeRetrievalNodeType } from './types'
 import { ALL_CHAT_AVAILABLE_BLOCKS, ALL_COMPLETION_AVAILABLE_BLOCKS } from '@/app/components/workflow/constants'
-
+import { DATASET_DEFAULT } from '@/config'
 import { RETRIEVE_TYPE } from '@/types/app'
 const i18nPrefix = 'workflow'
 
@@ -10,7 +10,12 @@ const nodeDefault: NodeDefault<KnowledgeRetrievalNodeType> = {
   defaultValue: {
     query_variable_selector: [],
     dataset_ids: [],
-    retrieval_mode: RETRIEVE_TYPE.oneWay,
+    retrieval_mode: RETRIEVE_TYPE.multiWay,
+    multiple_retrieval_config: {
+      top_k: DATASET_DEFAULT.top_k,
+      score_threshold: undefined,
+      reranking_enable: false,
+    },
   },
   getAvailablePrevNodes(isChatMode: boolean) {
     const nodes = isChatMode

web/app/components/workflow/nodes/knowledge-retrieval/hooks.ts

@@ -0,0 +1,14 @@
+import { useMemo } from 'react'
+import { getSelectedDatasetsMode } from './utils'
+import type {
+  DataSet,
+  SelectedDatasetsMode,
+} from '@/models/datasets'
+
+export const useSelectedDatasetsMode = (datasets: DataSet[]) => {
+  const selectedDatasetsMode: SelectedDatasetsMode = useMemo(() => {
+    return getSelectedDatasetsMode(datasets)
+  }, [datasets])
+
+  return selectedDatasetsMode
+}

web/app/components/workflow/nodes/knowledge-retrieval/panel.tsx

@@ -1,5 +1,8 @@
 import type { FC } from 'react'
-import React from 'react'
+import {
+  memo,
+  useCallback,
+} from 'react'
 import { useTranslation } from 'react-i18next'
 import VarReferencePicker from '../_base/components/variable/var-reference-picker'
 import useConfig from './use-config'
@@ -41,8 +44,14 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
     query,
     setQuery,
     runResult,
+    rerankModelOpen,
+    setRerankModelOpen,
   } = useConfig(id, data)
 
+  const handleOpenFromPropsChange = useCallback((openFromProps: boolean) => {
+    setRerankModelOpen(openFromProps)
+  }, [setRerankModelOpen])
+
   return (
     <div className='mt-2'>
       <div className='px-4 pb-4 space-y-4'>
@@ -75,7 +84,10 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
                 singleRetrievalModelConfig={inputs.single_retrieval_config?.model}
                 onSingleRetrievalModelChange={handleModelChanged as any}
                 onSingleRetrievalModelParamsChange={handleCompletionParamsChange}
-                readonly={readOnly}
+                readonly={readOnly || !selectedDatasets.length}
+                openFromProps={rerankModelOpen}
+                onOpenFromPropsChange={handleOpenFromPropsChange}
+                selectedDatasets={selectedDatasets}
               />
               {!readOnly && (<div className='w-px h-3 bg-gray-200'></div>)}
               {!readOnly && (
@@ -162,4 +174,4 @@ const Panel: FC<NodePanelProps<KnowledgeRetrievalNodeType>> = ({
   )
 }
 
-export default React.memo(Panel)
+export default memo(Panel)

web/app/components/workflow/nodes/knowledge-retrieval/types.ts

@@ -1,5 +1,9 @@
 import type { CommonNodeType, ModelConfig, ValueSelector } from '@/app/components/workflow/types'
 import type { RETRIEVE_TYPE } from '@/types/app'
+import type {
+  RerankingModeEnum,
+  WeightedScoreEnum,
+} from '@/models/datasets'
 
 export type MultipleRetrievalConfig = {
   top_k: number
@@ -8,6 +12,19 @@ export type MultipleRetrievalConfig = {
     provider: string
     model: string
   }
+  reranking_mode?: RerankingModeEnum
+  weights?: {
+    weight_type: WeightedScoreEnum
+    vector_setting: {
+      vector_weight: number
+      embedding_provider_name: string
+      embedding_model_name: string
+    }
+    keyword_setting: {
+      keyword_weight: number
+    }
+  }
+  reranking_enable?: boolean
 }
 
 export type SingleRetrievalConfig = {

web/app/components/workflow/nodes/knowledge-retrieval/use-config.ts

@@ -1,4 +1,9 @@
-import { useCallback, useEffect, useRef, useState } from 'react'
+import {
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+} from 'react'
 import produce from 'immer'
 import { isEqual } from 'lodash-es'
 import type { ValueSelector, Var } from '../../types'
@@ -8,6 +13,10 @@ import {
   useWorkflow,
 } from '../../hooks'
 import type { KnowledgeRetrievalNodeType, MultipleRetrievalConfig } from './types'
+import {
+  getMultipleRetrievalConfig,
+  getSelectedDatasetsMode,
+} from './utils'
 import { RETRIEVE_TYPE } from '@/types/app'
 import { DATASET_DEFAULT } from '@/config'
 import type { DataSet } from '@/models/datasets'
@@ -126,34 +135,20 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
       draft.multiple_retrieval_config = {
         top_k: multipleRetrievalConfig?.top_k || DATASET_DEFAULT.top_k,
         score_threshold: multipleRetrievalConfig?.score_threshold,
-        reranking_model: payload.retrieval_mode === RETRIEVE_TYPE.oneWay
-          ? undefined
-          : (!multipleRetrievalConfig?.reranking_model?.provider
-            ? {
-              provider: rerankDefaultModel?.provider?.provider || '',
-              model: rerankDefaultModel?.model || '',
-            }
-            : multipleRetrievalConfig?.reranking_model),
+        reranking_model: multipleRetrievalConfig?.reranking_model,
       }
     })
     setInputs(newInput)
   // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [currentProvider?.provider, currentModel, rerankDefaultModel])
-
+  const [selectedDatasets, setSelectedDatasets] = useState<DataSet[]>([])
+  const [rerankModelOpen, setRerankModelOpen] = useState(false)
   const handleRetrievalModeChange = useCallback((newMode: RETRIEVE_TYPE) => {
     const newInputs = produce(inputs, (draft) => {
       draft.retrieval_mode = newMode
       if (newMode === RETRIEVE_TYPE.multiWay) {
-        draft.multiple_retrieval_config = {
-          top_k: draft.multiple_retrieval_config?.top_k || DATASET_DEFAULT.top_k,
-          score_threshold: draft.multiple_retrieval_config?.score_threshold,
-          reranking_model: !draft.multiple_retrieval_config?.reranking_model?.provider
-            ? {
-              provider: rerankDefaultModel?.provider?.provider || '',
-              model: rerankDefaultModel?.model || '',
-            }
-            : draft.multiple_retrieval_config?.reranking_model,
-        }
+        const multipleRetrievalConfig = draft.multiple_retrieval_config
+        draft.multiple_retrieval_config = getMultipleRetrievalConfig(multipleRetrievalConfig!, selectedDatasets)
       }
       else {
         const hasSetModel = draft.single_retrieval_config?.model?.provider
@@ -170,17 +165,16 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
       }
     })
     setInputs(newInputs)
-  }, [currentModel?.model, currentModel?.model_properties?.mode, currentProvider?.provider, inputs, rerankDefaultModel?.model, rerankDefaultModel?.provider?.provider, setInputs])
+  }, [currentModel?.model, currentModel?.model_properties?.mode, currentProvider?.provider, inputs, setInputs, selectedDatasets])
 
   const handleMultipleRetrievalConfigChange = useCallback((newConfig: MultipleRetrievalConfig) => {
     const newInputs = produce(inputs, (draft) => {
-      draft.multiple_retrieval_config = newConfig
+      draft.multiple_retrieval_config = getMultipleRetrievalConfig(newConfig!, selectedDatasets)
     })
     setInputs(newInputs)
-  }, [inputs, setInputs])
+  }, [inputs, setInputs, selectedDatasets])
 
   // datasets
-  const [selectedDatasets, setSelectedDatasets] = useState<DataSet[]>([])
   useEffect(() => {
     (async () => {
       const inputs = inputRef.current
@@ -210,12 +204,25 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
   }, [])
 
   const handleOnDatasetsChange = useCallback((newDatasets: DataSet[]) => {
+    const {
+      allEconomic,
+      mixtureHighQualityAndEconomic,
+      inconsistentEmbeddingModel,
+    } = getSelectedDatasetsMode(newDatasets)
     const newInputs = produce(inputs, (draft) => {
       draft.dataset_ids = newDatasets.map(d => d.id)
+
+      if (payload.retrieval_mode === RETRIEVE_TYPE.multiWay && newDatasets.length > 0) {
+        const multipleRetrievalConfig = draft.multiple_retrieval_config
+        draft.multiple_retrieval_config = getMultipleRetrievalConfig(multipleRetrievalConfig!, newDatasets)
+      }
     })
     setInputs(newInputs)
     setSelectedDatasets(newDatasets)
-  }, [inputs, setInputs])
+
+    if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel)
+      setRerankModelOpen(true)
+  }, [inputs, setInputs, payload.retrieval_mode])
 
   const filterVar = useCallback((varPayload: Var) => {
     return varPayload.type === VarType.string
@@ -266,6 +273,8 @@ const useConfig = (id: string, payload: KnowledgeRetrievalNodeType) => {
     query,
     setQuery,
     runResult,
+    rerankModelOpen,
+    setRerankModelOpen,
   }
 }
 

web/app/components/workflow/nodes/knowledge-retrieval/utils.ts

@@ -1,5 +1,124 @@
-import type { KnowledgeRetrievalNodeType } from './types'
+import { uniq } from 'lodash-es'
+import type { MultipleRetrievalConfig } from './types'
+import type {
+  DataSet,
+  SelectedDatasetsMode,
+} from '@/models/datasets'
+import {
+  DEFAULT_WEIGHTED_SCORE,
+  RerankingModeEnum,
+  WeightedScoreEnum,
+} from '@/models/datasets'
+import { RETRIEVE_METHOD } from '@/types/app'
+import { DATASET_DEFAULT } from '@/config'
 
-export const checkNodeValid = (payload: KnowledgeRetrievalNodeType) => {
+export const checkNodeValid = () => {
   return true
 }
+
+export const getSelectedDatasetsMode = (datasets: DataSet[]) => {
+  let allHighQuality = true
+  let allHighQualityVectorSearch = true
+  let allHighQualityFullTextSearch = true
+  let allEconomic = true
+  let mixtureHighQualityAndEconomic = true
+  let inconsistentEmbeddingModel = false
+  if (!datasets.length) {
+    allHighQuality = false
+    allHighQualityVectorSearch = false
+    allHighQualityFullTextSearch = false
+    allEconomic = false
+    mixtureHighQualityAndEconomic = false
+    inconsistentEmbeddingModel = false
+  }
+  datasets.forEach((dataset) => {
+    if (dataset.indexing_technique === 'economy') {
+      allHighQuality = false
+      allHighQualityVectorSearch = false
+      allHighQualityFullTextSearch = false
+    }
+    if (dataset.indexing_technique === 'high_quality') {
+      allEconomic = false
+
+      if (dataset.retrieval_model_dict.search_method !== RETRIEVE_METHOD.semantic)
+        allHighQualityVectorSearch = false
+
+      if (dataset.retrieval_model_dict.search_method !== RETRIEVE_METHOD.fullText)
+        allHighQualityFullTextSearch = false
+    }
+  })
+
+  if (allHighQuality || allEconomic)
+    mixtureHighQualityAndEconomic = false
+
+  if (allHighQuality)
+    inconsistentEmbeddingModel = uniq(datasets.map(item => item.embedding_model)).length > 1
+
+  return {
+    allHighQuality,
+    allHighQualityVectorSearch,
+    allHighQualityFullTextSearch,
+    allEconomic,
+    mixtureHighQualityAndEconomic,
+    inconsistentEmbeddingModel,
+  } as SelectedDatasetsMode
+}
+
+export const getMultipleRetrievalConfig = (multipleRetrievalConfig: MultipleRetrievalConfig, selectedDatasets: DataSet[]) => {
+  const {
+    allHighQuality,
+    allHighQualityVectorSearch,
+    allHighQualityFullTextSearch,
+    allEconomic,
+    mixtureHighQualityAndEconomic,
+    inconsistentEmbeddingModel,
+  } = getSelectedDatasetsMode(selectedDatasets)
+
+  const {
+    top_k = DATASET_DEFAULT.top_k,
+    score_threshold,
+    reranking_mode,
+    reranking_model,
+    weights,
+    reranking_enable,
+  } = multipleRetrievalConfig || { top_k: DATASET_DEFAULT.top_k }
+
+  const result = {
+    top_k,
+    score_threshold,
+    reranking_mode,
+    reranking_model,
+    weights,
+    reranking_enable,
+  }
+
+  if (allEconomic || mixtureHighQualityAndEconomic || inconsistentEmbeddingModel)
+    result.reranking_mode = RerankingModeEnum.RerankingModel
+
+  if (allHighQuality && !inconsistentEmbeddingModel && reranking_mode === undefined)
+    result.reranking_mode = RerankingModeEnum.WeightedScore
+
+  if (allHighQuality && !inconsistentEmbeddingModel && (reranking_mode === RerankingModeEnum.WeightedScore || reranking_mode === undefined) && !weights) {
+    result.weights = {
+      weight_type: WeightedScoreEnum.Customized,
+      vector_setting: {
+        vector_weight: allHighQualityVectorSearch
+          ? DEFAULT_WEIGHTED_SCORE.allHighQualityVectorSearch.semantic
+          : allHighQualityFullTextSearch
+            ? DEFAULT_WEIGHTED_SCORE.allHighQualityFullTextSearch.semantic
+            : DEFAULT_WEIGHTED_SCORE.other.semantic,
+        embedding_provider_name: selectedDatasets[0].embedding_model_provider,
+        embedding_model_name: selectedDatasets[0].embedding_model,
+      },
+      keyword_setting: {
+        keyword_weight: allHighQualityVectorSearch
+          ? DEFAULT_WEIGHTED_SCORE.allHighQualityVectorSearch.keyword
+          : allHighQualityFullTextSearch
+            ? DEFAULT_WEIGHTED_SCORE.allHighQualityFullTextSearch.keyword
+            : DEFAULT_WEIGHTED_SCORE.other.keyword,
+      },
+    }
+  }
+
+  return result
+}

web/config/index.ts

@@ -138,8 +138,8 @@ export const appDefaultIconBackground = '#D5F5F6'
 export const NEED_REFRESH_APP_LIST_KEY = 'needRefreshAppList'
 
 export const DATASET_DEFAULT = {
-  top_k: 2,
-  score_threshold: 0.5,
+  top_k: 4,
+  score_threshold: 0.8,
 }
 
 export const APP_PAGE_LIMIT = 10

web/context/debug-configuration.ts

@@ -97,6 +97,8 @@ type IDebugConfiguration = {
   isShowVisionConfig: boolean
   visionConfig: VisionSettings
   setVisionConfig: (visionConfig: VisionSettings, noNotice?: boolean) => void
+  rerankSettingModalOpen: boolean
+  setRerankSettingModalOpen: (rerankSettingModalOpen: boolean) => void
 }
 
 const DebugConfigurationContext = createContext<IDebugConfiguration>({
@@ -217,7 +219,7 @@ const DebugConfigurationContext = createContext<IDebugConfiguration>({
   showSelectDataSet: () => { },
   setDataSets: () => { },
   datasetConfigs: {
-    retrieval_model: RETRIEVE_TYPE.oneWay,
+    retrieval_model: RETRIEVE_TYPE.multiWay,
     reranking_model: {
       reranking_provider_name: '',
       reranking_model_name: '',
@@ -239,6 +241,8 @@ const DebugConfigurationContext = createContext<IDebugConfiguration>({
     transfer_methods: [TransferMethod.remote_url],
   },
   setVisionConfig: () => { },
+  rerankSettingModalOpen: false,
+  setRerankSettingModalOpen: () => { },
 })
 
 export const useDebugConfigurationContext = () => useContext(DebugConfigurationContext)

web/hooks/use-knowledge.ts

@@ -0,0 +1,29 @@
+import { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+
+export const useKnowledge = () => {
+  const { t } = useTranslation()
+
+  const formatIndexingTechnique = useCallback((indexingTechnique: string) => {
+    return t(`dataset.indexingTechnique.${indexingTechnique}`)
+  }, [t])
+
+  const formatIndexingMethod = useCallback((indexingMethod: string) => {
+    return t(`dataset.indexingMethod.${indexingMethod}`)
+  }, [t])
+
+  const formatIndexingTechniqueAndMethod = useCallback((indexingTechnique: string, indexingMethod: string) => {
+    let result = formatIndexingTechnique(indexingTechnique)
+
+    if (indexingMethod)
+      result += ` · ${formatIndexingMethod(indexingMethod)}`
+
+    return result
+  }, [formatIndexingTechnique, formatIndexingMethod])
+
+  return {
+    formatIndexingTechnique,
+    formatIndexingMethod,
+    formatIndexingTechniqueAndMethod,
+  }
+}

web/i18n/en-US/dataset.ts

@@ -45,6 +45,29 @@ const translation = {
   },
   docsFailedNotice: 'documents failed to be indexed',
   retry: 'Retry',
+  indexingTechnique: {
+    high_quality: 'HQ',
+    economy: 'ECO',
+  },
+  indexingMethod: {
+    semantic_search: 'VECTOR',
+    full_text_search: 'FULL TEXT',
+    hybrid_search: 'HYBRID',
+  },
+  mixtureHighQualityAndEconomicTip: 'The Rerank model is required for mixture of high quality and economical knowledge bases.',
+  inconsistentEmbeddingModelTip: 'The Rerank model is required if the Embedding models of the selected knowledge bases are inconsistent.',
+  retrievalSettings: 'Retrieval Setting',
+  rerankSettings: 'Rerank Setting',
+  weightedScore: {
+    title: 'Weighted Score',
+    description: 'By adjusting the weights assigned， this rerank strategy determines whether to prioritize semantic or keyword matching.',
+    semanticFirst: 'Semantic first',
+    keywordFirst: 'Keyword first',
+    customized: 'Customized',
+    semantic: 'Semantic',
+    keyword: 'Keyword',
+  },
+  nTo1RetrievalLegacy: 'According to product planning, N-to-1 retrieval will be officially deprecated in September. Until then you can still use it normally.',
 }
 
 export default translation

web/i18n/zh-Hans/dataset.ts

@@ -45,6 +45,29 @@ const translation = {
   },
   docsFailedNotice: '文档无法被索引',
   retry: '重试',
+  indexingTechnique: {
+    high_quality: '高质量',
+    economy: '经济',
+  },
+  indexingMethod: {
+    semantic_search: '向量检索',
+    full_text_search: '全文检索',
+    hybrid_search: '混合检索',
+  },
+  mixtureHighQualityAndEconomicTip: '混合使用高质量和经济型知识库需要配置 Rerank 模型。',
+  inconsistentEmbeddingModelTip: '当所选知识库配置的 Embedding 模型不一致时，需要配置 Rerank 模型。',
+  retrievalSettings: '召回设置',
+  rerankSettings: 'Rerank 设置',
+  weightedScore: {
+    title: '权重设置',
+    description: '通过调整分配的权重，重新排序策略确定是优先进行语义匹配还是关键字匹配。',
+    semanticFirst: '语义优先',
+    keywordFirst: '关键词优先',
+    customized: '自定义',
+    semantic: '语义',
+    keyword: '关键词',
+  },
+  nTo1RetrievalLegacy: '根据产品规划，N 选 1 召回将于 9 月正式弃用。在那之前，您仍然可以正常使用它。',
 }
 
 export default translation

web/models/datasets.ts

@@ -449,3 +449,46 @@ export type ErrorDocsResponse = {
   data: IndexingStatusResponse[]
   total: number
 }
+
+export type SelectedDatasetsMode = {
+  allHighQuality: boolean
+  allHighQualityVectorSearch: boolean
+  allHighQualityFullTextSearch: boolean
+  allEconomic: boolean
+  mixtureHighQualityAndEconomic: boolean
+  inconsistentEmbeddingModel: boolean
+}
+
+export enum WeightedScoreEnum {
+  SemanticFirst = 'semantic_first',
+  KeywordFirst = 'keyword_first',
+  Customized = 'customized',
+}
+
+export enum RerankingModeEnum {
+  RerankingModel = 'reranking_model',
+  WeightedScore = 'weighted_score',
+}
+
+export const DEFAULT_WEIGHTED_SCORE = {
+  allHighQualityVectorSearch: {
+    semantic: 1.0,
+    keyword: 0,
+  },
+  allHighQualityFullTextSearch: {
+    semantic: 0,
+    keyword: 1.0,
+  },
+  semanticFirst: {
+    semantic: 0.7,
+    keyword: 0.3,
+  },
+  keywordFirst: {
+    semantic: 0.3,
+    keyword: 0.7,
+  },
+  other: {
+    semantic: 0.7,
+    keyword: 0.3,
+  },
+}

web/models/debug.ts

@@ -1,4 +1,8 @@
 import type { AgentStrategy, ModelModeType, RETRIEVE_TYPE, ToolItem, TtsAutoPlay } from '@/types/app'
+import type {
+  RerankingModeEnum,
+  WeightedScoreEnum,
+} from '@/models/datasets'
 export type Inputs = Record<string, string | number | object>
 
 export enum PromptMode {
@@ -144,13 +148,26 @@ export type DatasetConfigs = {
   }
   top_k: number
   score_threshold_enabled: boolean
-  score_threshold?: number | null
+  score_threshold: number | null | undefined
   datasets: {
     datasets: {
       enabled: boolean
       id: string
     }[]
   }
+  reranking_mode?: RerankingModeEnum
+  weights?: {
+    weight_type: WeightedScoreEnum
+    vector_setting: {
+      vector_weight: number
+      embedding_provider_name: string
+      embedding_model_name: string
+    }
+    keyword_setting: {
+      keyword_weight: number
+    }
+  }
+  reranking_enable?: boolean
 }
 
 export type DebugRequestBody = {

web/types/app.ts

@@ -2,6 +2,10 @@ import type { AnnotationReplyConfig, ChatPromptConfig, CompletionPromptConfig, D
 import type { CollectionType } from '@/app/components/tools/types'
 import type { LanguagesSupported } from '@/i18n/language'
 import type { Tag } from '@/app/components/base/tag-management/constant'
+import type {
+  RerankingModeEnum,
+  WeightedScoreEnum,
+} from '@/models/datasets'
 
 export enum Theme {
   light = 'light',
@@ -403,4 +407,16 @@ export type RetrievalConfig = {
   top_k: number
   score_threshold_enabled: boolean
   score_threshold: number
+  reranking_mode?: RerankingModeEnum
+  weights?: {
+    weight_type: WeightedScoreEnum
+    vector_setting: {
+      vector_weight: number
+      embedding_provider_name: string
+      embedding_model_name: string
+    }
+    keyword_setting: {
+      keyword_weight: number
+    }
+  }
 }