feat: bump version to 0.3.24 (#1262)

Co-authored-by: jyong <jyong@dify.ai>

.github/ISSUE_TEMPLATE/help_wanted.yml

@@ -0,0 +1,11 @@
+name: "🤝 Help Wanted"
+description: "Request help from the community"
+labels:
+- help-wanted
+body:
+- type: textarea
+  attributes:
+    label: Provide a description of the help you need
+    placeholder: Briefly describe what you need help with.
+  validations:
+    required: true

api/.env.example

@@ -50,24 +50,6 @@ S3_REGION=your-region
 WEB_API_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 CONSOLE_CORS_ALLOW_ORIGINS=http://127.0.0.1:3000,*
 
-# Cookie configuration
-COOKIE_HTTPONLY=true
-COOKIE_SAMESITE=None
-COOKIE_SECURE=true
-
-# Session configuration
-SESSION_PERMANENT=true
-SESSION_USE_SIGNER=true
-
-## support redis, sqlalchemy
-SESSION_TYPE=redis
-
-# session redis configuration
-SESSION_REDIS_HOST=localhost
-SESSION_REDIS_PORT=6379
-SESSION_REDIS_PASSWORD=difyai123456
-SESSION_REDIS_DB=2
-
 # Vector database configuration, support: weaviate, qdrant
 VECTOR_STORE=weaviate
 

api/app.py

@@ -1,8 +1,7 @@
 # -*- coding:utf-8 -*-
 import os
-from datetime import datetime, timedelta
 
-from werkzeug.exceptions import Forbidden
+from werkzeug.exceptions import Unauthorized
 
 if not os.environ.get("DEBUG") or os.environ.get("DEBUG").lower() != 'true':
     from gevent import monkey
@@ -12,12 +11,11 @@ import logging
 import json
 import threading
 
-from flask import Flask, request, Response, session
-import flask_login
+from flask import Flask, request, Response
 from flask_cors import CORS
 
 from core.model_providers.providers import hosted
-from extensions import ext_session, ext_celery, ext_sentry, ext_redis, ext_login, ext_migrate, \
+from extensions import ext_celery, ext_sentry, ext_redis, ext_login, ext_migrate, \
     ext_database, ext_storage, ext_mail, ext_stripe
 from extensions.ext_database import db
 from extensions.ext_login import login_manager
@@ -27,12 +25,10 @@ from models import model, account, dataset, web, task, source, tool
 from events import event_handlers
 # DO NOT REMOVE ABOVE
 
-import core
 from config import Config, CloudEditionConfig
 from commands import register_commands
-from models.account import TenantAccountJoin, AccountStatus
-from models.model import Account, EndUser, App
-from services.account_service import TenantService
+from services.account_service import AccountService
+from libs.passport import PassportService
 
 import warnings
 warnings.simplefilter("ignore", ResourceWarning)
@@ -85,81 +81,33 @@ def initialize_extensions(app):
     ext_redis.init_app(app)
     ext_storage.init_app(app)
     ext_celery.init_app(app)
-    ext_session.init_app(app)
     ext_login.init_app(app)
     ext_mail.init_app(app)
     ext_sentry.init_app(app)
     ext_stripe.init_app(app)
 
 
-def _create_tenant_for_account(account):
-    tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
-
-    TenantService.create_tenant_member(tenant, account, role='owner')
-    account.current_tenant = tenant
-
-    return tenant
-
-
 # Flask-Login configuration
-@login_manager.user_loader
-def load_user(user_id):
-    """Load user based on the user_id."""
+@login_manager.request_loader
+def load_user_from_request(request_from_flask_login):
+    """Load user based on the request."""
     if request.blueprint == 'console':
         # Check if the user_id contains a dot, indicating the old format
-        if '.' in user_id:
-            tenant_id, account_id = user_id.split('.')
-        else:
-            account_id = user_id
+        auth_header = request.headers.get('Authorization', '')
+        if ' ' not in auth_header:
+            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+        auth_scheme, auth_token = auth_header.split(None, 1)
+        auth_scheme = auth_scheme.lower()
+        if auth_scheme != 'bearer':
+            raise Unauthorized('Invalid Authorization header format. Expected \'Bearer <api-key>\' format.')
+        
+        decoded = PassportService().verify(auth_token)
+        user_id = decoded.get('user_id')
 
-        account = db.session.query(Account).filter(Account.id == account_id).first()
-
-        if account:
-            if account.status == AccountStatus.BANNED.value or account.status == AccountStatus.CLOSED.value:
-                raise Forbidden('Account is banned or closed.')
-
-            workspace_id = session.get('workspace_id')
-            if workspace_id:
-                tenant_account_join = db.session.query(TenantAccountJoin).filter(
-                    TenantAccountJoin.account_id == account.id,
-                    TenantAccountJoin.tenant_id == workspace_id
-                ).first()
-
-                if not tenant_account_join:
-                    tenant_account_join = db.session.query(TenantAccountJoin).filter(
-                        TenantAccountJoin.account_id == account.id).first()
-
-                    if tenant_account_join:
-                        account.current_tenant_id = tenant_account_join.tenant_id
-                    else:
-                        _create_tenant_for_account(account)
-                    session['workspace_id'] = account.current_tenant_id
-                else:
-                    account.current_tenant_id = workspace_id
-            else:
-                tenant_account_join = db.session.query(TenantAccountJoin).filter(
-                    TenantAccountJoin.account_id == account.id).first()
-                if tenant_account_join:
-                    account.current_tenant_id = tenant_account_join.tenant_id
-                else:
-                    _create_tenant_for_account(account)
-                session['workspace_id'] = account.current_tenant_id
-
-            current_time = datetime.utcnow()
-
-            # update last_active_at when last_active_at is more than 10 minutes ago
-            if current_time - account.last_active_at > timedelta(minutes=10):
-                account.last_active_at = current_time
-                db.session.commit()
-
-            # Log in the user with the updated user_id
-            flask_login.login_user(account, remember=True)
-
-        return account
+        return AccountService.load_user(user_id)
     else:
         return None
 
-
 @login_manager.unauthorized_handler
 def unauthorized_handler():
     """Handle unauthorized requests."""
@@ -216,6 +164,7 @@ if app.config['TESTING']:
 @app.after_request
 def after_request(response):
     """Add Version headers to the response."""
+    response.set_cookie('remember_token', '', expires=0)
     response.headers.add('X-Version', app.config['CURRENT_VERSION'])
     response.headers.add('X-Env', app.config['DEPLOY_ENV'])
     return response

api/commands.py

@@ -3,12 +3,13 @@ import json
 import math
 import random
 import string
+import threading
 import time
 import uuid
 
 import click
 from tqdm import tqdm
-from flask import current_app
+from flask import current_app, Flask
 from langchain.embeddings import OpenAIEmbeddings
 from werkzeug.exceptions import NotFound
 
@@ -456,92 +457,92 @@ def update_qdrant_indexes():
 @click.command('normalization-collections', help='restore all collections in one')
 def normalization_collections():
     click.echo(click.style('Start normalization collections.', fg='green'))
-    normalization_count = 0
-
+    normalization_count = []
     page = 1
     while True:
         try:
             datasets = db.session.query(Dataset).filter(Dataset.indexing_technique == 'high_quality') \
-                .order_by(Dataset.created_at.desc()).paginate(page=page, per_page=50)
+                .order_by(Dataset.created_at.desc()).paginate(page=page, per_page=100)
         except NotFound:
             break
-
+        datasets_result = datasets.items
         page += 1
-        for dataset in datasets:
-            if not dataset.collection_binding_id:
-                try:
-                    click.echo('restore dataset index: {}'.format(dataset.id))
-                    try:
-                        embedding_model = ModelFactory.get_embedding_model(
-                            tenant_id=dataset.tenant_id,
-                            model_provider_name=dataset.embedding_model_provider,
-                            model_name=dataset.embedding_model
-                        )
-                    except Exception:
-                        provider = Provider(
-                            id='provider_id',
-                            tenant_id=dataset.tenant_id,
-                            provider_name='openai',
-                            provider_type=ProviderType.CUSTOM.value,
-                            encrypted_config=json.dumps({'openai_api_key': 'TEST'}),
-                            is_valid=True,
-                        )
-                        model_provider = OpenAIProvider(provider=provider)
-                        embedding_model = OpenAIEmbedding(name="text-embedding-ada-002",
-                                                          model_provider=model_provider)
-                    embeddings = CacheEmbedding(embedding_model)
-                    dataset_collection_binding = db.session.query(DatasetCollectionBinding). \
-                        filter(DatasetCollectionBinding.provider_name == embedding_model.model_provider.provider_name,
-                               DatasetCollectionBinding.model_name == embedding_model.name). \
-                        order_by(DatasetCollectionBinding.created_at). \
-                        first()
+        for i in range(0, len(datasets_result), 5):
+            threads = []
+            sub_datasets = datasets_result[i:i + 5]
+            for dataset in sub_datasets:
+                document_format_thread = threading.Thread(target=deal_dataset_vector, kwargs={
+                    'flask_app': current_app._get_current_object(),
+                    'dataset': dataset,
+                    'normalization_count': normalization_count
+                })
+                threads.append(document_format_thread)
+                document_format_thread.start()
+            for thread in threads:
+                thread.join()
 
-                    if not dataset_collection_binding:
-                        dataset_collection_binding = DatasetCollectionBinding(
-                            provider_name=embedding_model.model_provider.provider_name,
-                            model_name=embedding_model.name,
-                            collection_name="Vector_index_" + str(uuid.uuid4()).replace("-", "_") + '_Node'
-                        )
-                        db.session.add(dataset_collection_binding)
-                        db.session.commit()
+    click.echo(click.style('Congratulations! restore {} dataset indexes.'.format(len(normalization_count)), fg='green'))
 
-                    from core.index.vector_index.qdrant_vector_index import QdrantVectorIndex, QdrantConfig
 
-                    index = QdrantVectorIndex(
-                        dataset=dataset,
-                        config=QdrantConfig(
-                            endpoint=current_app.config.get('QDRANT_URL'),
-                            api_key=current_app.config.get('QDRANT_API_KEY'),
-                            root_path=current_app.root_path
-                        ),
-                        embeddings=embeddings
-                    )
-                    if index:
-                        index.restore_dataset_in_one(dataset, dataset_collection_binding)
-                    else:
-                        click.echo('passed.')
+def deal_dataset_vector(flask_app: Flask, dataset: Dataset, normalization_count: list):
+    with flask_app.app_context():
+        try:
+            click.echo('restore dataset index: {}'.format(dataset.id))
+            try:
+                embedding_model = ModelFactory.get_embedding_model(
+                    tenant_id=dataset.tenant_id,
+                    model_provider_name=dataset.embedding_model_provider,
+                    model_name=dataset.embedding_model
+                )
+            except Exception:
+                provider = Provider(
+                    id='provider_id',
+                    tenant_id=dataset.tenant_id,
+                    provider_name='openai',
+                    provider_type=ProviderType.CUSTOM.value,
+                    encrypted_config=json.dumps({'openai_api_key': 'TEST'}),
+                    is_valid=True,
+                )
+                model_provider = OpenAIProvider(provider=provider)
+                embedding_model = OpenAIEmbedding(name="text-embedding-ada-002",
+                                                  model_provider=model_provider)
+            embeddings = CacheEmbedding(embedding_model)
+            dataset_collection_binding = db.session.query(DatasetCollectionBinding). \
+                filter(DatasetCollectionBinding.provider_name == embedding_model.model_provider.provider_name,
+                       DatasetCollectionBinding.model_name == embedding_model.name). \
+                order_by(DatasetCollectionBinding.created_at). \
+                first()
 
-                    original_index = QdrantVectorIndex(
-                        dataset=dataset,
-                        config=QdrantConfig(
-                            endpoint=current_app.config.get('QDRANT_URL'),
-                            api_key=current_app.config.get('QDRANT_API_KEY'),
-                            root_path=current_app.root_path
-                        ),
-                        embeddings=embeddings
-                    )
-                    if original_index:
-                        original_index.delete_original_collection(dataset, dataset_collection_binding)
-                        normalization_count += 1
-                    else:
-                        click.echo('passed.')
-                except Exception as e:
-                    click.echo(
-                        click.style('Create dataset index error: {} {}'.format(e.__class__.__name__, str(e)),
-                                    fg='red'))
-                    continue
+            if not dataset_collection_binding:
+                dataset_collection_binding = DatasetCollectionBinding(
+                    provider_name=embedding_model.model_provider.provider_name,
+                    model_name=embedding_model.name,
+                    collection_name="Vector_index_" + str(uuid.uuid4()).replace("-", "_") + '_Node'
+                )
+                db.session.add(dataset_collection_binding)
+                db.session.commit()
 
-    click.echo(click.style('Congratulations! restore {} dataset indexes.'.format(normalization_count), fg='green'))
+            from core.index.vector_index.qdrant_vector_index import QdrantVectorIndex, QdrantConfig
+
+            index = QdrantVectorIndex(
+                dataset=dataset,
+                config=QdrantConfig(
+                    endpoint=current_app.config.get('QDRANT_URL'),
+                    api_key=current_app.config.get('QDRANT_API_KEY'),
+                    root_path=current_app.root_path
+                ),
+                embeddings=embeddings
+            )
+            if index:
+                # index.delete_by_group_id(dataset.id)
+                index.restore_dataset_in_one(dataset, dataset_collection_binding)
+            else:
+                click.echo('passed.')
+            normalization_count.append(1)
+        except Exception as e:
+            click.echo(
+                click.style('Create dataset index error: {} {}'.format(e.__class__.__name__, str(e)),
+                            fg='red'))
 
 
 @click.command('update_app_model_configs', help='Migrate data to support paragraph variable.')
@@ -646,6 +647,76 @@ def update_app_model_configs(batch_size):
 
             pbar.update(len(data_batch))
 
+@click.command('migrate_default_input_to_dataset_query_variable')
+@click.option("--batch-size", default=500, help="Number of records to migrate in each batch.")
+def migrate_default_input_to_dataset_query_variable(batch_size):
+
+    click.secho("Starting...", fg='green')
+
+    total_records = db.session.query(AppModelConfig) \
+        .join(App, App.app_model_config_id == AppModelConfig.id) \
+        .filter(App.mode == 'completion') \
+        .filter(AppModelConfig.dataset_query_variable == None) \
+        .count()
+    
+    if total_records == 0:
+        click.secho("No data to migrate.", fg='green')
+        return
+
+    num_batches = (total_records + batch_size - 1) // batch_size
+    
+    with tqdm(total=total_records, desc="Migrating Data") as pbar:
+        for i in range(num_batches):
+            offset = i * batch_size
+            limit = min(batch_size, total_records - offset)
+
+            click.secho(f"Fetching batch {i + 1}/{num_batches} from source database...", fg='green')
+
+            data_batch = db.session.query(AppModelConfig) \
+                .join(App, App.app_model_config_id == AppModelConfig.id) \
+                .filter(App.mode == 'completion') \
+                .filter(AppModelConfig.dataset_query_variable == None) \
+                .order_by(App.created_at) \
+                .offset(offset).limit(limit).all()
+
+            if not data_batch:
+                click.secho("No more data to migrate.", fg='green')
+                break
+
+            try:
+                click.secho(f"Migrating {len(data_batch)} records...", fg='green')
+                for data in data_batch:
+                    config = AppModelConfig.to_dict(data)
+
+                    tools = config["agent_mode"]["tools"]
+                    dataset_exists = "dataset" in str(tools)
+                    if not dataset_exists:
+                        continue
+
+                    user_input_form = config.get("user_input_form", [])
+                    for form in user_input_form:
+                        paragraph = form.get('paragraph')
+                        if paragraph \
+                            and paragraph.get('variable') == 'query':
+                                data.dataset_query_variable = 'query'
+                                break
+                        
+                        if paragraph \
+                            and paragraph.get('variable') == 'default_input':
+                                data.dataset_query_variable = 'default_input'
+                                break
+
+                db.session.commit()
+
+            except Exception as e:
+                click.secho(f"Error while migrating data: {e}, app_id: {data.app_id}, app_model_config_id: {data.id}",
+                            fg='red')
+                continue
+            
+            click.secho(f"Successfully migrated batch {i + 1}/{num_batches}.", fg='green')
+
+            pbar.update(len(data_batch))
+
 
 def register_commands(app):
     app.cli.add_command(reset_password)
@@ -659,3 +730,4 @@ def register_commands(app):
     app.cli.add_command(update_qdrant_indexes)
     app.cli.add_command(update_app_model_configs)
     app.cli.add_command(normalization_collections)
+    app.cli.add_command(migrate_default_input_to_dataset_query_variable)

api/config.py

@@ -10,9 +10,6 @@ from extensions.ext_redis import redis_client
 dotenv.load_dotenv()
 
 DEFAULTS = {
-    'COOKIE_HTTPONLY': 'True',
-    'COOKIE_SECURE': 'True',
-    'COOKIE_SAMESITE': 'None',
     'DB_USERNAME': 'postgres',
     'DB_PASSWORD': '',
     'DB_HOST': 'localhost',
@@ -22,10 +19,6 @@ DEFAULTS = {
     'REDIS_PORT': '6379',
     'REDIS_DB': '0',
     'REDIS_USE_SSL': 'False',
-    'SESSION_REDIS_HOST': 'localhost',
-    'SESSION_REDIS_PORT': '6379',
-    'SESSION_REDIS_DB': '2',
-    'SESSION_REDIS_USE_SSL': 'False',
     'OAUTH_REDIRECT_PATH': '/console/api/oauth/authorize',
     'OAUTH_REDIRECT_INDEX_PATH': '/',
     'CONSOLE_WEB_URL': 'https://cloud.dify.ai',
@@ -36,9 +29,6 @@ DEFAULTS = {
     'STORAGE_TYPE': 'local',
     'STORAGE_LOCAL_PATH': 'storage',
     'CHECK_UPDATE_URL': 'https://updates.dify.ai',
-    'SESSION_TYPE': 'sqlalchemy',
-    'SESSION_PERMANENT': 'True',
-    'SESSION_USE_SIGNER': 'True',
     'DEPLOY_ENV': 'PRODUCTION',
     'SQLALCHEMY_POOL_SIZE': 30,
     'SQLALCHEMY_POOL_RECYCLE': 3600,
@@ -102,7 +92,7 @@ class Config:
         self.CONSOLE_URL = get_env('CONSOLE_URL')
         self.API_URL = get_env('API_URL')
         self.APP_URL = get_env('APP_URL')
-        self.CURRENT_VERSION = "0.3.23"
+        self.CURRENT_VERSION = "0.3.24"
         self.COMMIT_SHA = get_env('COMMIT_SHA')
         self.EDITION = "SELF_HOSTED"
         self.DEPLOY_ENV = get_env('DEPLOY_ENV')
@@ -115,20 +105,6 @@ class Config:
         # Alternatively you can set it with `SECRET_KEY` environment variable.
         self.SECRET_KEY = get_env('SECRET_KEY')
 
-        # cookie settings
-        self.REMEMBER_COOKIE_HTTPONLY = get_bool_env('COOKIE_HTTPONLY')
-        self.SESSION_COOKIE_HTTPONLY = get_bool_env('COOKIE_HTTPONLY')
-        self.REMEMBER_COOKIE_SAMESITE = get_env('COOKIE_SAMESITE')
-        self.SESSION_COOKIE_SAMESITE = get_env('COOKIE_SAMESITE')
-        self.REMEMBER_COOKIE_SECURE = get_bool_env('COOKIE_SECURE')
-        self.SESSION_COOKIE_SECURE = get_bool_env('COOKIE_SECURE')
-        self.PERMANENT_SESSION_LIFETIME = timedelta(days=7)
-
-        # session settings, only support sqlalchemy, redis
-        self.SESSION_TYPE = get_env('SESSION_TYPE')
-        self.SESSION_PERMANENT = get_bool_env('SESSION_PERMANENT')
-        self.SESSION_USE_SIGNER = get_bool_env('SESSION_USE_SIGNER')
-
         # redis settings
         self.REDIS_HOST = get_env('REDIS_HOST')
         self.REDIS_PORT = get_env('REDIS_PORT')
@@ -137,14 +113,6 @@ class Config:
         self.REDIS_DB = get_env('REDIS_DB')
         self.REDIS_USE_SSL = get_bool_env('REDIS_USE_SSL')
 
-        # session redis settings
-        self.SESSION_REDIS_HOST = get_env('SESSION_REDIS_HOST')
-        self.SESSION_REDIS_PORT = get_env('SESSION_REDIS_PORT')
-        self.SESSION_REDIS_USERNAME = get_env('SESSION_REDIS_USERNAME')
-        self.SESSION_REDIS_PASSWORD = get_env('SESSION_REDIS_PASSWORD')
-        self.SESSION_REDIS_DB = get_env('SESSION_REDIS_DB')
-        self.SESSION_REDIS_USE_SSL = get_bool_env('SESSION_REDIS_USE_SSL')
-
         # storage settings
         self.STORAGE_TYPE = get_env('STORAGE_TYPE')
         self.STORAGE_LOCAL_PATH = get_env('STORAGE_LOCAL_PATH')

api/controllers/console/apikey.py

@@ -81,6 +81,7 @@ class BaseApiKeyListResource(Resource):
         key = ApiToken.generate_api_key(self.token_prefix, 24)
         api_token = ApiToken()
         setattr(api_token, self.resource_id_field, resource_id)
+        api_token.tenant_id = current_user.current_tenant_id
         api_token.token = key
         api_token.type = self.resource_type
         db.session.add(api_token)

api/controllers/console/app/app.py

@@ -19,40 +19,13 @@ from core.model_providers.model_factory import ModelFactory
 from core.model_providers.model_provider_factory import ModelProviderFactory
 from core.model_providers.models.entity.model_params import ModelType
 from events.app_event import app_was_created, app_was_deleted
+from fields.app_fields import app_pagination_fields, app_detail_fields, template_list_fields, \
+    app_detail_fields_with_site
 from libs.helper import TimestampField
 from extensions.ext_database import db
 from models.model import App, AppModelConfig, Site
 from services.app_model_config_service import AppModelConfigService
 
-model_config_fields = {
-    'opening_statement': fields.String,
-    'suggested_questions': fields.Raw(attribute='suggested_questions_list'),
-    'suggested_questions_after_answer': fields.Raw(attribute='suggested_questions_after_answer_dict'),
-    'speech_to_text': fields.Raw(attribute='speech_to_text_dict'),
-    'retriever_resource': fields.Raw(attribute='retriever_resource_dict'),
-    'more_like_this': fields.Raw(attribute='more_like_this_dict'),
-    'sensitive_word_avoidance': fields.Raw(attribute='sensitive_word_avoidance_dict'),
-    'model': fields.Raw(attribute='model_dict'),
-    'user_input_form': fields.Raw(attribute='user_input_form_list'),
-    'pre_prompt': fields.String,
-    'agent_mode': fields.Raw(attribute='agent_mode_dict'),
-}
-
-app_detail_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'mode': fields.String,
-    'icon': fields.String,
-    'icon_background': fields.String,
-    'enable_site': fields.Boolean,
-    'enable_api': fields.Boolean,
-    'api_rpm': fields.Integer,
-    'api_rph': fields.Integer,
-    'is_demo': fields.Boolean,
-    'model_config': fields.Nested(model_config_fields, attribute='app_model_config'),
-    'created_at': TimestampField
-}
-
 
 def _get_app(app_id, tenant_id):
     app = db.session.query(App).filter(App.id == app_id, App.tenant_id == tenant_id).first()
@@ -62,35 +35,6 @@ def _get_app(app_id, tenant_id):
 
 
 class AppListApi(Resource):
-    prompt_config_fields = {
-        'prompt_template': fields.String,
-    }
-
-    model_config_partial_fields = {
-        'model': fields.Raw(attribute='model_dict'),
-        'pre_prompt': fields.String,
-    }
-
-    app_partial_fields = {
-        'id': fields.String,
-        'name': fields.String,
-        'mode': fields.String,
-        'icon': fields.String,
-        'icon_background': fields.String,
-        'enable_site': fields.Boolean,
-        'enable_api': fields.Boolean,
-        'is_demo': fields.Boolean,
-        'model_config': fields.Nested(model_config_partial_fields, attribute='app_model_config'),
-        'created_at': TimestampField
-    }
-
-    app_pagination_fields = {
-        'page': fields.Integer,
-        'limit': fields.Integer(attribute='per_page'),
-        'total': fields.Integer,
-        'has_more': fields.Boolean(attribute='has_next'),
-        'data': fields.List(fields.Nested(app_partial_fields), attribute='items')
-    }
 
     @setup_required
     @login_required
@@ -162,7 +106,8 @@ class AppListApi(Resource):
             model_configuration = AppModelConfigService.validate_configuration(
                 tenant_id=current_user.current_tenant_id,
                 account=current_user,
-                config=model_config_dict
+                config=model_config_dict,
+                mode=args['mode']
             )
 
             app = App(
@@ -236,18 +181,6 @@ class AppListApi(Resource):
 
 
 class AppTemplateApi(Resource):
-    template_fields = {
-        'name': fields.String,
-        'icon': fields.String,
-        'icon_background': fields.String,
-        'description': fields.String,
-        'mode': fields.String,
-        'model_config': fields.Nested(model_config_fields),
-    }
-
-    template_list_fields = {
-        'data': fields.List(fields.Nested(template_fields)),
-    }
 
     @setup_required
     @login_required
@@ -266,38 +199,6 @@ class AppTemplateApi(Resource):
 
 
 class AppApi(Resource):
-    site_fields = {
-        'access_token': fields.String(attribute='code'),
-        'code': fields.String,
-        'title': fields.String,
-        'icon': fields.String,
-        'icon_background': fields.String,
-        'description': fields.String,
-        'default_language': fields.String,
-        'customize_domain': fields.String,
-        'copyright': fields.String,
-        'privacy_policy': fields.String,
-        'customize_token_strategy': fields.String,
-        'prompt_public': fields.Boolean,
-        'app_base_url': fields.String,
-    }
-
-    app_detail_fields_with_site = {
-        'id': fields.String,
-        'name': fields.String,
-        'mode': fields.String,
-        'icon': fields.String,
-        'icon_background': fields.String,
-        'enable_site': fields.Boolean,
-        'enable_api': fields.Boolean,
-        'api_rpm': fields.Integer,
-        'api_rph': fields.Integer,
-        'is_demo': fields.Boolean,
-        'model_config': fields.Nested(model_config_fields, attribute='app_model_config'),
-        'site': fields.Nested(site_fields),
-        'api_base_url': fields.String,
-        'created_at': TimestampField
-    }
 
     @setup_required
     @login_required

api/controllers/console/app/conversation.py

@@ -13,107 +13,14 @@ from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
+from fields.conversation_fields import conversation_pagination_fields, conversation_detail_fields, \
+    conversation_message_detail_fields, conversation_with_summary_pagination_fields
 from libs.helper import TimestampField, datetime_string, uuid_value
 from extensions.ext_database import db
 from models.model import Message, MessageAnnotation, Conversation
 
-account_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'email': fields.String
-}
-
-feedback_fields = {
-    'rating': fields.String,
-    'content': fields.String,
-    'from_source': fields.String,
-    'from_end_user_id': fields.String,
-    'from_account': fields.Nested(account_fields, allow_null=True),
-}
-
-annotation_fields = {
-    'content': fields.String,
-    'account': fields.Nested(account_fields, allow_null=True),
-    'created_at': TimestampField
-}
-
-message_detail_fields = {
-    'id': fields.String,
-    'conversation_id': fields.String,
-    'inputs': fields.Raw,
-    'query': fields.String,
-    'message': fields.Raw,
-    'message_tokens': fields.Integer,
-    'answer': fields.String,
-    'answer_tokens': fields.Integer,
-    'provider_response_latency': fields.Float,
-    'from_source': fields.String,
-    'from_end_user_id': fields.String,
-    'from_account_id': fields.String,
-    'feedbacks': fields.List(fields.Nested(feedback_fields)),
-    'annotation': fields.Nested(annotation_fields, allow_null=True),
-    'created_at': TimestampField
-}
-
-feedback_stat_fields = {
-    'like': fields.Integer,
-    'dislike': fields.Integer
-}
-
-model_config_fields = {
-    'opening_statement': fields.String,
-    'suggested_questions': fields.Raw,
-    'model': fields.Raw,
-    'user_input_form': fields.Raw,
-    'pre_prompt': fields.String,
-    'agent_mode': fields.Raw,
-}
-
 
 class CompletionConversationApi(Resource):
-    class MessageTextField(fields.Raw):
-        def format(self, value):
-            return value[0]['text'] if value else ''
-
-    simple_configs_fields = {
-        'prompt_template': fields.String,
-    }
-
-    simple_model_config_fields = {
-        'model': fields.Raw(attribute='model_dict'),
-        'pre_prompt': fields.String,
-    }
-
-    simple_message_detail_fields = {
-        'inputs': fields.Raw,
-        'query': fields.String,
-        'message': MessageTextField,
-        'answer': fields.String,
-    }
-
-    conversation_fields = {
-        'id': fields.String,
-        'status': fields.String,
-        'from_source': fields.String,
-        'from_end_user_id': fields.String,
-        'from_end_user_session_id': fields.String(),
-        'from_account_id': fields.String,
-        'read_at': TimestampField,
-        'created_at': TimestampField,
-        'annotation': fields.Nested(annotation_fields, allow_null=True),
-        'model_config': fields.Nested(simple_model_config_fields),
-        'user_feedback_stats': fields.Nested(feedback_stat_fields),
-        'admin_feedback_stats': fields.Nested(feedback_stat_fields),
-        'message': fields.Nested(simple_message_detail_fields, attribute='first_message')
-    }
-
-    conversation_pagination_fields = {
-        'page': fields.Integer,
-        'limit': fields.Integer(attribute='per_page'),
-        'total': fields.Integer,
-        'has_more': fields.Boolean(attribute='has_next'),
-        'data': fields.List(fields.Nested(conversation_fields), attribute='items')
-    }
 
     @setup_required
     @login_required
@@ -191,21 +98,11 @@ class CompletionConversationApi(Resource):
 
 
 class CompletionConversationDetailApi(Resource):
-    conversation_detail_fields = {
-        'id': fields.String,
-        'status': fields.String,
-        'from_source': fields.String,
-        'from_end_user_id': fields.String,
-        'from_account_id': fields.String,
-        'created_at': TimestampField,
-        'model_config': fields.Nested(model_config_fields),
-        'message': fields.Nested(message_detail_fields, attribute='first_message'),
-    }
 
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(conversation_detail_fields)
+    @marshal_with(conversation_message_detail_fields)
     def get(self, app_id, conversation_id):
         app_id = str(app_id)
         conversation_id = str(conversation_id)
@@ -234,44 +131,11 @@ class CompletionConversationDetailApi(Resource):
 
 
 class ChatConversationApi(Resource):
-    simple_configs_fields = {
-        'prompt_template': fields.String,
-    }
-
-    simple_model_config_fields = {
-        'model': fields.Raw(attribute='model_dict'),
-        'pre_prompt': fields.String,
-    }
-
-    conversation_fields = {
-        'id': fields.String,
-        'status': fields.String,
-        'from_source': fields.String,
-        'from_end_user_id': fields.String,
-        'from_end_user_session_id': fields.String,
-        'from_account_id': fields.String,
-        'summary': fields.String(attribute='summary_or_query'),
-        'read_at': TimestampField,
-        'created_at': TimestampField,
-        'annotated': fields.Boolean,
-        'model_config': fields.Nested(simple_model_config_fields),
-        'message_count': fields.Integer,
-        'user_feedback_stats': fields.Nested(feedback_stat_fields),
-        'admin_feedback_stats': fields.Nested(feedback_stat_fields)
-    }
-
-    conversation_pagination_fields = {
-        'page': fields.Integer,
-        'limit': fields.Integer(attribute='per_page'),
-        'total': fields.Integer,
-        'has_more': fields.Boolean(attribute='has_next'),
-        'data': fields.List(fields.Nested(conversation_fields), attribute='items')
-    }
 
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(conversation_pagination_fields)
+    @marshal_with(conversation_with_summary_pagination_fields)
     def get(self, app_id):
         app_id = str(app_id)
 
@@ -356,19 +220,6 @@ class ChatConversationApi(Resource):
 
 
 class ChatConversationDetailApi(Resource):
-    conversation_detail_fields = {
-        'id': fields.String,
-        'status': fields.String,
-        'from_source': fields.String,
-        'from_end_user_id': fields.String,
-        'from_account_id': fields.String,
-        'created_at': TimestampField,
-        'annotated': fields.Boolean,
-        'model_config': fields.Nested(model_config_fields),
-        'message_count': fields.Integer,
-        'user_feedback_stats': fields.Nested(feedback_stat_fields),
-        'admin_feedback_stats': fields.Nested(feedback_stat_fields)
-    }
 
     @setup_required
     @login_required

api/controllers/console/app/message.py

@@ -17,6 +17,7 @@ from controllers.console.wraps import account_initialization_required
 from core.model_providers.error import LLMRateLimitError, LLMBadRequestError, LLMAuthorizationError, LLMAPIConnectionError, \
     ProviderTokenNotInitError, LLMAPIUnavailableError, QuotaExceededError, ModelCurrentlyNotSupportError
 from core.login.login import login_required
+from fields.conversation_fields import message_detail_fields
 from libs.helper import uuid_value, TimestampField
 from libs.infinite_scroll_pagination import InfiniteScrollPagination
 from extensions.ext_database import db
@@ -27,44 +28,6 @@ from services.errors.conversation import ConversationNotExistsError
 from services.errors.message import MessageNotExistsError
 from services.message_service import MessageService
 
-account_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'email': fields.String
-}
-
-feedback_fields = {
-    'rating': fields.String,
-    'content': fields.String,
-    'from_source': fields.String,
-    'from_end_user_id': fields.String,
-    'from_account': fields.Nested(account_fields, allow_null=True),
-}
-
-annotation_fields = {
-    'content': fields.String,
-    'account': fields.Nested(account_fields, allow_null=True),
-    'created_at': TimestampField
-}
-
-message_detail_fields = {
-    'id': fields.String,
-    'conversation_id': fields.String,
-    'inputs': fields.Raw,
-    'query': fields.String,
-    'message': fields.Raw,
-    'message_tokens': fields.Integer,
-    'answer': fields.String,
-    'answer_tokens': fields.Integer,
-    'provider_response_latency': fields.Float,
-    'from_source': fields.String,
-    'from_end_user_id': fields.String,
-    'from_account_id': fields.String,
-    'feedbacks': fields.List(fields.Nested(feedback_fields)),
-    'annotation': fields.Nested(annotation_fields, allow_null=True),
-    'created_at': TimestampField
-}
-
 
 class ChatMessageListApi(Resource):
     message_infinite_scroll_pagination_fields = {

api/controllers/console/app/model_config.py

@@ -31,7 +31,8 @@ class ModelConfigResource(Resource):
         model_configuration = AppModelConfigService.validate_configuration(
             tenant_id=current_user.current_tenant_id,
             account=current_user,
-            config=request.json
+            config=request.json,
+            mode=app_model.mode
         )
 
         new_app_model_config = AppModelConfig(

api/controllers/console/app/site.py

@@ -8,26 +8,11 @@ from controllers.console import api
 from controllers.console.app import _get_app
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
+from fields.app_fields import app_site_fields
 from libs.helper import supported_language
 from extensions.ext_database import db
 from models.model import Site
 
-app_site_fields = {
-    'app_id': fields.String,
-    'access_token': fields.String(attribute='code'),
-    'code': fields.String,
-    'title': fields.String,
-    'icon': fields.String,
-    'icon_background': fields.String,
-    'description': fields.String,
-    'default_language': fields.String,
-    'customize_domain': fields.String,
-    'copyright': fields.String,
-    'privacy_policy': fields.String,
-    'customize_token_strategy': fields.String,
-    'prompt_public': fields.Boolean
-}
-
 
 def parse_app_site_args():
     parser = reqparse.RequestParser()

api/controllers/console/auth/data_source_oauth.py

@@ -45,15 +45,34 @@ class OAuthDataSource(Resource):
         if current_app.config.get('NOTION_INTEGRATION_TYPE') == 'internal':
             internal_secret = current_app.config.get('NOTION_INTERNAL_SECRET')
             oauth_provider.save_internal_access_token(internal_secret)
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?oauth_data_source=success')
+            return { 'data': '' }
         else:
             auth_url = oauth_provider.get_authorization_url()
-            return redirect(auth_url)
+            return { 'data': auth_url }, 200
 
 
 
 
 class OAuthDataSourceCallback(Resource):
+    def get(self, provider: str):
+        OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers()
+        with current_app.app_context():
+            oauth_provider = OAUTH_DATASOURCE_PROVIDERS.get(provider)
+        if not oauth_provider:
+            return {'error': 'Invalid provider'}, 400
+        if 'code' in request.args:
+            code = request.args.get('code')
+
+            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&code={code}')
+        elif 'error' in request.args:
+            error = request.args.get('error')
+
+            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error={error}')
+        else:
+            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?type=notion&error=Access denied')
+        
+
+class OAuthDataSourceBinding(Resource):
     def get(self, provider: str):
         OAUTH_DATASOURCE_PROVIDERS = get_oauth_providers()
         with current_app.app_context():
@@ -69,12 +88,7 @@ class OAuthDataSourceCallback(Resource):
                     f"An error occurred during the OAuthCallback process with {provider}: {e.response.text}")
                 return {'error': 'OAuth data source process failed'}, 400
 
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?oauth_data_source=success')
-        elif 'error' in request.args:
-            error = request.args.get('error')
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?oauth_data_source={error}')
-        else:
-            return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?oauth_data_source=access_denied')
+            return {'result': 'success'}, 200
 
 
 class OAuthDataSourceSync(Resource):
@@ -101,4 +115,5 @@ class OAuthDataSourceSync(Resource):
 
 api.add_resource(OAuthDataSource, '/oauth/data-source/<string:provider>')
 api.add_resource(OAuthDataSourceCallback, '/oauth/data-source/callback/<string:provider>')
+api.add_resource(OAuthDataSourceBinding, '/oauth/data-source/binding/<string:provider>')
 api.add_resource(OAuthDataSourceSync, '/oauth/data-source/<string:provider>/<uuid:binding_id>/sync')

api/controllers/console/auth/login.py

@@ -6,7 +6,6 @@ from flask_restful import Resource, reqparse
 
 import services
 from controllers.console import api
-from controllers.console.error import AccountNotLinkTenantError
 from controllers.console.setup import setup_required
 from libs.helper import email
 from libs.password import valid_password
@@ -37,12 +36,12 @@ class LoginApi(Resource):
         except Exception:
             pass
 
-        flask_login.login_user(account, remember=args['remember_me'])
         AccountService.update_last_login(account, request)
 
         # todo: return the user info
+        token = AccountService.get_account_jwt_token(account)
 
-        return {'result': 'success'}
+        return {'result': 'success', 'data': token}
 
 
 class LogoutApi(Resource):

api/controllers/console/auth/oauth.py

@@ -2,9 +2,8 @@ import logging
 from datetime import datetime
 from typing import Optional
 
-import flask_login
 import requests
-from flask import request, redirect, current_app, session
+from flask import request, redirect, current_app
 from flask_restful import Resource
 
 from libs.oauth import OAuthUserInfo, GitHubOAuth, GoogleOAuth
@@ -75,12 +74,11 @@ class OAuthCallback(Resource):
             account.initialized_at = datetime.utcnow()
             db.session.commit()
 
-        # login user
-        session.clear()
-        flask_login.login_user(account, remember=True)
         AccountService.update_last_login(account, request)
 
-        return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?oauth_login=success')
+        token = AccountService.get_account_jwt_token(account)
+
+        return redirect(f'{current_app.config.get("CONSOLE_WEB_URL")}?console_token={token}')
 
 
 def _get_account_by_openid_or_email(provider: str, user_info: OAuthUserInfo) -> Optional[Account]:

api/controllers/console/datasets/data_source.py

@@ -14,6 +14,7 @@ from controllers.console.wraps import account_initialization_required
 from core.data_loader.loader.notion import NotionLoader
 from core.indexing_runner import IndexingRunner
 from extensions.ext_database import db
+from fields.data_source_fields import integrate_notion_info_list_fields, integrate_list_fields
 from libs.helper import TimestampField
 from models.dataset import Document
 from models.source import DataSourceBinding
@@ -24,37 +25,6 @@ cache = TTLCache(maxsize=None, ttl=30)
 
 
 class DataSourceApi(Resource):
-    integrate_icon_fields = {
-        'type': fields.String,
-        'url': fields.String,
-        'emoji': fields.String
-    }
-    integrate_page_fields = {
-        'page_name': fields.String,
-        'page_id': fields.String,
-        'page_icon': fields.Nested(integrate_icon_fields, allow_null=True),
-        'parent_id': fields.String,
-        'type': fields.String
-    }
-    integrate_workspace_fields = {
-        'workspace_name': fields.String,
-        'workspace_id': fields.String,
-        'workspace_icon': fields.String,
-        'pages': fields.List(fields.Nested(integrate_page_fields)),
-        'total': fields.Integer
-    }
-    integrate_fields = {
-        'id': fields.String,
-        'provider': fields.String,
-        'created_at': TimestampField,
-        'is_bound': fields.Boolean,
-        'disabled': fields.Boolean,
-        'link': fields.String,
-        'source_info': fields.Nested(integrate_workspace_fields)
-    }
-    integrate_list_fields = {
-        'data': fields.List(fields.Nested(integrate_fields)),
-    }
 
     @setup_required
     @login_required
@@ -131,28 +101,6 @@ class DataSourceApi(Resource):
 
 
 class DataSourceNotionListApi(Resource):
-    integrate_icon_fields = {
-        'type': fields.String,
-        'url': fields.String,
-        'emoji': fields.String
-    }
-    integrate_page_fields = {
-        'page_name': fields.String,
-        'page_id': fields.String,
-        'page_icon': fields.Nested(integrate_icon_fields, allow_null=True),
-        'is_bound': fields.Boolean,
-        'parent_id': fields.String,
-        'type': fields.String
-    }
-    integrate_workspace_fields = {
-        'workspace_name': fields.String,
-        'workspace_id': fields.String,
-        'workspace_icon': fields.String,
-        'pages': fields.List(fields.Nested(integrate_page_fields))
-    }
-    integrate_notion_info_list_fields = {
-        'notion_info': fields.List(fields.Nested(integrate_workspace_fields)),
-    }
 
     @setup_required
     @login_required

api/controllers/console/datasets/datasets.py

@@ -1,6 +1,9 @@
 # -*- coding:utf-8 -*-
-from flask import request
+import flask_restful
+from flask import request, current_app
 from flask_login import current_user
+
+from controllers.console.apikey import api_key_list, api_key_fields
 from core.login.login import login_required
 from flask_restful import Resource, reqparse, fields, marshal, marshal_with
 from werkzeug.exceptions import NotFound, Forbidden
@@ -12,45 +15,16 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.indexing_runner import IndexingRunner
 from core.model_providers.error import LLMBadRequestError, ProviderTokenNotInitError
-from core.model_providers.model_factory import ModelFactory
 from core.model_providers.models.entity.model_params import ModelType
-from libs.helper import TimestampField
+from fields.app_fields import related_app_list
+from fields.dataset_fields import dataset_detail_fields, dataset_query_detail_fields
+from fields.document_fields import document_status_fields
 from extensions.ext_database import db
 from models.dataset import DocumentSegment, Document
-from models.model import UploadFile
+from models.model import UploadFile, ApiToken
 from services.dataset_service import DatasetService, DocumentService
 from services.provider_service import ProviderService
 
-dataset_detail_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'description': fields.String,
-    'provider': fields.String,
-    'permission': fields.String,
-    'data_source_type': fields.String,
-    'indexing_technique': fields.String,
-    'app_count': fields.Integer,
-    'document_count': fields.Integer,
-    'word_count': fields.Integer,
-    'created_by': fields.String,
-    'created_at': TimestampField,
-    'updated_by': fields.String,
-    'updated_at': TimestampField,
-    'embedding_model': fields.String,
-    'embedding_model_provider': fields.String,
-    'embedding_available': fields.Boolean
-}
-
-dataset_query_detail_fields = {
-    "id": fields.String,
-    "content": fields.String,
-    "source": fields.String,
-    "source_app_id": fields.String,
-    "created_by_role": fields.String,
-    "created_by": fields.String,
-    "created_at": TimestampField
-}
-
 
 def _validate_name(name):
     if not name or len(name) < 1 or len(name) > 40:
@@ -82,7 +56,8 @@ class DatasetListApi(Resource):
 
         # check embedding setting
         provider_service = ProviderService()
-        valid_model_list = provider_service.get_valid_model_list(current_user.current_tenant_id, ModelType.EMBEDDINGS.value)
+        valid_model_list = provider_service.get_valid_model_list(current_user.current_tenant_id,
+                                                                 ModelType.EMBEDDINGS.value)
         # if len(valid_model_list) == 0:
         #     raise ProviderNotInitializeError(
         #         f"No Embedding Model available. Please configure a valid provider "
@@ -157,7 +132,8 @@ class DatasetApi(Resource):
         # check embedding setting
         provider_service = ProviderService()
         # get valid model list
-        valid_model_list = provider_service.get_valid_model_list(current_user.current_tenant_id, ModelType.EMBEDDINGS.value)
+        valid_model_list = provider_service.get_valid_model_list(current_user.current_tenant_id,
+                                                                 ModelType.EMBEDDINGS.value)
         model_names = []
         for valid_model in valid_model_list:
             model_names.append(f"{valid_model['model_name']}:{valid_model['model_provider']['provider_name']}")
@@ -271,7 +247,8 @@ class DatasetIndexingEstimateApi(Resource):
         parser.add_argument('indexing_technique', type=str, required=True, nullable=True, location='json')
         parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
         parser.add_argument('dataset_id', type=str, required=False, nullable=False, location='json')
-        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False, location='json')
+        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False,
+                            location='json')
         args = parser.parse_args()
         # validate args
         DocumentService.estimate_args_validate(args)
@@ -320,18 +297,6 @@ class DatasetIndexingEstimateApi(Resource):
 
 
 class DatasetRelatedAppListApi(Resource):
-    app_detail_kernel_fields = {
-        'id': fields.String,
-        'name': fields.String,
-        'mode': fields.String,
-        'icon': fields.String,
-        'icon_background': fields.String,
-    }
-
-    related_app_list = {
-        'data': fields.List(fields.Nested(app_detail_kernel_fields)),
-        'total': fields.Integer,
-    }
 
     @setup_required
     @login_required
@@ -363,24 +328,6 @@ class DatasetRelatedAppListApi(Resource):
 
 
 class DatasetIndexingStatusApi(Resource):
-    document_status_fields = {
-        'id': fields.String,
-        'indexing_status': fields.String,
-        'processing_started_at': TimestampField,
-        'parsing_completed_at': TimestampField,
-        'cleaning_completed_at': TimestampField,
-        'splitting_completed_at': TimestampField,
-        'completed_at': TimestampField,
-        'paused_at': TimestampField,
-        'error': fields.String,
-        'stopped_at': TimestampField,
-        'completed_segments': fields.Integer,
-        'total_segments': fields.Integer,
-    }
-
-    document_status_fields_list = {
-        'data': fields.List(fields.Nested(document_status_fields))
-    }
 
     @setup_required
     @login_required
@@ -400,16 +347,101 @@ class DatasetIndexingStatusApi(Resource):
                                                           DocumentSegment.status != 're_segment').count()
             document.completed_segments = completed_segments
             document.total_segments = total_segments
-            documents_status.append(marshal(document, self.document_status_fields))
+            documents_status.append(marshal(document, document_status_fields))
         data = {
             'data': documents_status
         }
         return data
 
 
+class DatasetApiKeyApi(Resource):
+    max_keys = 10
+    token_prefix = 'dataset-'
+    resource_type = 'dataset'
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(api_key_list)
+    def get(self):
+        keys = db.session.query(ApiToken). \
+            filter(ApiToken.type == self.resource_type, ApiToken.tenant_id == current_user.current_tenant_id). \
+            all()
+        return {"items": keys}
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(api_key_fields)
+    def post(self):
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        current_key_count = db.session.query(ApiToken). \
+            filter(ApiToken.type == self.resource_type, ApiToken.tenant_id == current_user.current_tenant_id). \
+            count()
+
+        if current_key_count >= self.max_keys:
+            flask_restful.abort(
+                400,
+                message=f"Cannot create more than {self.max_keys} API keys for this resource type.",
+                code='max_keys_exceeded'
+            )
+
+        key = ApiToken.generate_api_key(self.token_prefix, 24)
+        api_token = ApiToken()
+        api_token.tenant_id = current_user.current_tenant_id
+        api_token.token = key
+        api_token.type = self.resource_type
+        db.session.add(api_token)
+        db.session.commit()
+        return api_token, 200
+
+
+class DatasetApiDeleteApi(Resource):
+    resource_type = 'dataset'
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def delete(self, api_key_id):
+        api_key_id = str(api_key_id)
+
+        # The role of the current user in the ta table must be admin or owner
+        if current_user.current_tenant.current_role not in ['admin', 'owner']:
+            raise Forbidden()
+
+        key = db.session.query(ApiToken). \
+            filter(ApiToken.tenant_id == current_user.current_tenant_id, ApiToken.type == self.resource_type,
+                   ApiToken.id == api_key_id). \
+            first()
+
+        if key is None:
+            flask_restful.abort(404, message='API key not found')
+
+        db.session.query(ApiToken).filter(ApiToken.id == api_key_id).delete()
+        db.session.commit()
+
+        return {'result': 'success'}, 204
+
+
+class DatasetApiBaseUrlApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def get(self):
+        return {
+            'api_base_url': (current_app.config['SERVICE_API_URL'] if current_app.config['SERVICE_API_URL']
+                             else request.host_url.rstrip('/')) + '/v1'
+        }
+
+
 api.add_resource(DatasetListApi, '/datasets')
 api.add_resource(DatasetApi, '/datasets/<uuid:dataset_id>')
 api.add_resource(DatasetQueryApi, '/datasets/<uuid:dataset_id>/queries')
 api.add_resource(DatasetIndexingEstimateApi, '/datasets/indexing-estimate')
 api.add_resource(DatasetRelatedAppListApi, '/datasets/<uuid:dataset_id>/related-apps')
 api.add_resource(DatasetIndexingStatusApi, '/datasets/<uuid:dataset_id>/indexing-status')
+api.add_resource(DatasetApiKeyApi, '/datasets/api-keys')
+api.add_resource(DatasetApiDeleteApi, '/datasets/api-keys/<uuid:api_key_id>')
+api.add_resource(DatasetApiBaseUrlApi, '/datasets/api-base-info')

api/controllers/console/datasets/datasets_document.py

@@ -23,6 +23,8 @@ from core.model_providers.error import ProviderTokenNotInitError, QuotaExceededE
     LLMBadRequestError
 from core.model_providers.model_factory import ModelFactory
 from extensions.ext_redis import redis_client
+from fields.document_fields import document_with_segments_fields, document_fields, \
+    dataset_and_document_fields, document_status_fields
 from libs.helper import TimestampField
 from extensions.ext_database import db
 from models.dataset import DatasetProcessRule, Dataset
@@ -32,64 +34,6 @@ from services.dataset_service import DocumentService, DatasetService
 from tasks.add_document_to_index_task import add_document_to_index_task
 from tasks.remove_document_from_index_task import remove_document_from_index_task
 
-dataset_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'description': fields.String,
-    'permission': fields.String,
-    'data_source_type': fields.String,
-    'indexing_technique': fields.String,
-    'created_by': fields.String,
-    'created_at': TimestampField,
-}
-
-document_fields = {
-    'id': fields.String,
-    'position': fields.Integer,
-    'data_source_type': fields.String,
-    'data_source_info': fields.Raw(attribute='data_source_info_dict'),
-    'dataset_process_rule_id': fields.String,
-    'name': fields.String,
-    'created_from': fields.String,
-    'created_by': fields.String,
-    'created_at': TimestampField,
-    'tokens': fields.Integer,
-    'indexing_status': fields.String,
-    'error': fields.String,
-    'enabled': fields.Boolean,
-    'disabled_at': TimestampField,
-    'disabled_by': fields.String,
-    'archived': fields.Boolean,
-    'display_status': fields.String,
-    'word_count': fields.Integer,
-    'hit_count': fields.Integer,
-    'doc_form': fields.String,
-}
-
-document_with_segments_fields = {
-    'id': fields.String,
-    'position': fields.Integer,
-    'data_source_type': fields.String,
-    'data_source_info': fields.Raw(attribute='data_source_info_dict'),
-    'dataset_process_rule_id': fields.String,
-    'name': fields.String,
-    'created_from': fields.String,
-    'created_by': fields.String,
-    'created_at': TimestampField,
-    'tokens': fields.Integer,
-    'indexing_status': fields.String,
-    'error': fields.String,
-    'enabled': fields.Boolean,
-    'disabled_at': TimestampField,
-    'disabled_by': fields.String,
-    'archived': fields.Boolean,
-    'display_status': fields.String,
-    'word_count': fields.Integer,
-    'hit_count': fields.Integer,
-    'completed_segments': fields.Integer,
-    'total_segments': fields.Integer
-}
-
 
 class DocumentResource(Resource):
     def get_document(self, dataset_id: str, document_id: str) -> Document:
@@ -303,11 +247,6 @@ class DatasetDocumentListApi(Resource):
 
 
 class DatasetInitApi(Resource):
-    dataset_and_document_fields = {
-        'dataset': fields.Nested(dataset_fields),
-        'documents': fields.List(fields.Nested(document_fields)),
-        'batch': fields.String
-    }
 
     @setup_required
     @login_required
@@ -504,24 +443,6 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
 
 
 class DocumentBatchIndexingStatusApi(DocumentResource):
-    document_status_fields = {
-        'id': fields.String,
-        'indexing_status': fields.String,
-        'processing_started_at': TimestampField,
-        'parsing_completed_at': TimestampField,
-        'cleaning_completed_at': TimestampField,
-        'splitting_completed_at': TimestampField,
-        'completed_at': TimestampField,
-        'paused_at': TimestampField,
-        'error': fields.String,
-        'stopped_at': TimestampField,
-        'completed_segments': fields.Integer,
-        'total_segments': fields.Integer,
-    }
-
-    document_status_fields_list = {
-        'data': fields.List(fields.Nested(document_status_fields))
-    }
 
     @setup_required
     @login_required
@@ -541,7 +462,7 @@ class DocumentBatchIndexingStatusApi(DocumentResource):
             document.total_segments = total_segments
             if document.is_paused:
                 document.indexing_status = 'paused'
-            documents_status.append(marshal(document, self.document_status_fields))
+            documents_status.append(marshal(document, document_status_fields))
         data = {
             'data': documents_status
         }
@@ -549,20 +470,6 @@ class DocumentBatchIndexingStatusApi(DocumentResource):
 
 
 class DocumentIndexingStatusApi(DocumentResource):
-    document_status_fields = {
-        'id': fields.String,
-        'indexing_status': fields.String,
-        'processing_started_at': TimestampField,
-        'parsing_completed_at': TimestampField,
-        'cleaning_completed_at': TimestampField,
-        'splitting_completed_at': TimestampField,
-        'completed_at': TimestampField,
-        'paused_at': TimestampField,
-        'error': fields.String,
-        'stopped_at': TimestampField,
-        'completed_segments': fields.Integer,
-        'total_segments': fields.Integer,
-    }
 
     @setup_required
     @login_required
@@ -586,7 +493,7 @@ class DocumentIndexingStatusApi(DocumentResource):
         document.total_segments = total_segments
         if document.is_paused:
             document.indexing_status = 'paused'
-        return marshal(document, self.document_status_fields)
+        return marshal(document, document_status_fields)
 
 
 class DocumentDetailApi(DocumentResource):

api/controllers/console/datasets/datasets_segments.py

@@ -3,7 +3,7 @@ import uuid
 from datetime import datetime
 from flask import request
 from flask_login import current_user
-from flask_restful import Resource, reqparse, fields, marshal
+from flask_restful import Resource, reqparse, marshal
 from werkzeug.exceptions import NotFound, Forbidden
 
 import services
@@ -17,6 +17,7 @@ from core.model_providers.model_factory import ModelFactory
 from core.login.login import login_required
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
+from fields.segment_fields import segment_fields
 from models.dataset import DocumentSegment
 
 from libs.helper import TimestampField
@@ -26,36 +27,6 @@ from tasks.disable_segment_from_index_task import disable_segment_from_index_tas
 from tasks.batch_create_segment_to_index_task import batch_create_segment_to_index_task
 import pandas as pd
 
-segment_fields = {
-    'id': fields.String,
-    'position': fields.Integer,
-    'document_id': fields.String,
-    'content': fields.String,
-    'answer': fields.String,
-    'word_count': fields.Integer,
-    'tokens': fields.Integer,
-    'keywords': fields.List(fields.String),
-    'index_node_id': fields.String,
-    'index_node_hash': fields.String,
-    'hit_count': fields.Integer,
-    'enabled': fields.Boolean,
-    'disabled_at': TimestampField,
-    'disabled_by': fields.String,
-    'status': fields.String,
-    'created_by': fields.String,
-    'created_at': TimestampField,
-    'indexing_at': TimestampField,
-    'completed_at': TimestampField,
-    'error': fields.String,
-    'stopped_at': TimestampField
-}
-
-segment_list_response = {
-    'data': fields.List(fields.Nested(segment_fields)),
-    'has_more': fields.Boolean,
-    'limit': fields.Integer
-}
-
 
 class DatasetDocumentSegmentListApi(Resource):
     @setup_required

api/controllers/console/datasets/file.py

@@ -1,28 +1,19 @@
-import datetime
-import hashlib
-import tempfile
-import chardet
-import time
-import uuid
-from pathlib import Path
-
 from cachetools import TTLCache
 from flask import request, current_app
-from flask_login import current_user
+
+import services
 from core.login.login import login_required
 from flask_restful import Resource, marshal_with, fields
-from werkzeug.exceptions import NotFound
 
 from controllers.console import api
 from controllers.console.datasets.error import NoFileUploadedError, TooManyFilesError, FileTooLargeError, \
     UnsupportedFileTypeError
+
 from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
-from core.data_loader.file_extractor import FileExtractor
-from extensions.ext_storage import storage
-from libs.helper import TimestampField
-from extensions.ext_database import db
-from models.model import UploadFile
+from fields.file_fields import upload_config_fields, file_fields
+
+from services.file_service import FileService
 
 cache = TTLCache(maxsize=None, ttl=30)
 
@@ -31,10 +22,6 @@ PREVIEW_WORDS_LIMIT = 3000
 
 
 class FileApi(Resource):
-    upload_config_fields = {
-        'file_size_limit': fields.Integer,
-        'batch_count_limit': fields.Integer
-    }
 
     @setup_required
     @login_required
@@ -48,16 +35,6 @@ class FileApi(Resource):
             'batch_count_limit': batch_count_limit
         }, 200
 
-    file_fields = {
-        'id': fields.String,
-        'name': fields.String,
-        'size': fields.Integer,
-        'extension': fields.String,
-        'mime_type': fields.String,
-        'created_by': fields.String,
-        'created_at': TimestampField,
-    }
-
     @setup_required
     @login_required
     @account_initialization_required
@@ -73,45 +50,13 @@ class FileApi(Resource):
 
         if len(request.files) > 1:
             raise TooManyFilesError()
-
-        file_content = file.read()
-        file_size = len(file_content)
-
-        file_size_limit = current_app.config.get("UPLOAD_FILE_SIZE_LIMIT") * 1024 * 1024
-        if file_size > file_size_limit:
-            message = "({file_size} > {file_size_limit})"
-            raise FileTooLargeError(message)
-
-        extension = file.filename.split('.')[-1]
-        if extension.lower() not in ALLOWED_EXTENSIONS:
+        try:
+            upload_file = FileService.upload_file(file)
+        except services.errors.file.FileTooLargeError as file_too_large_error:
+            raise FileTooLargeError(file_too_large_error.description)
+        except services.errors.file.UnsupportedFileTypeError:
             raise UnsupportedFileTypeError()
 
-        # user uuid as file name
-        file_uuid = str(uuid.uuid4())
-        file_key = 'upload_files/' + current_user.current_tenant_id + '/' + file_uuid + '.' + extension
-
-        # save file to storage
-        storage.save(file_key, file_content)
-
-        # save file to db
-        config = current_app.config
-        upload_file = UploadFile(
-            tenant_id=current_user.current_tenant_id,
-            storage_type=config['STORAGE_TYPE'],
-            key=file_key,
-            name=file.filename,
-            size=file_size,
-            extension=extension,
-            mime_type=file.mimetype,
-            created_by=current_user.id,
-            created_at=datetime.datetime.utcnow(),
-            used=False,
-            hash=hashlib.sha3_256(file_content).hexdigest()
-        )
-
-        db.session.add(upload_file)
-        db.session.commit()
-
         return upload_file, 201
 
 
@@ -121,26 +66,7 @@ class FilePreviewApi(Resource):
     @account_initialization_required
     def get(self, file_id):
         file_id = str(file_id)
-
-        key = file_id + request.path
-        cached_response = cache.get(key)
-        if cached_response and time.time() - cached_response['timestamp'] < cache.ttl:
-            return cached_response['response']
-
-        upload_file = db.session.query(UploadFile) \
-            .filter(UploadFile.id == file_id) \
-            .first()
-
-        if not upload_file:
-            raise NotFound("File not found")
-
-        # extract text from file
-        extension = upload_file.extension
-        if extension.lower() not in ALLOWED_EXTENSIONS:
-            raise UnsupportedFileTypeError()
-
-        text = FileExtractor.load(upload_file, return_text=True)
-        text = text[0:PREVIEW_WORDS_LIMIT] if text else ''
+        text = FileService.get_file_preview(file_id)
         return {'content': text}
 
 

api/controllers/console/datasets/hit_testing.py

@@ -2,7 +2,7 @@ import logging
 
 from flask_login import current_user
 from core.login.login import login_required
-from flask_restful import Resource, reqparse, marshal, fields
+from flask_restful import Resource, reqparse, marshal
 from werkzeug.exceptions import InternalServerError, NotFound, Forbidden
 
 import services
@@ -14,48 +14,10 @@ from controllers.console.setup import setup_required
 from controllers.console.wraps import account_initialization_required
 from core.model_providers.error import ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError, \
     LLMBadRequestError
-from libs.helper import TimestampField
+from fields.hit_testing_fields import hit_testing_record_fields
 from services.dataset_service import DatasetService
 from services.hit_testing_service import HitTestingService
 
-document_fields = {
-    'id': fields.String,
-    'data_source_type': fields.String,
-    'name': fields.String,
-    'doc_type': fields.String,
-}
-
-segment_fields = {
-    'id': fields.String,
-    'position': fields.Integer,
-    'document_id': fields.String,
-    'content': fields.String,
-    'answer': fields.String,
-    'word_count': fields.Integer,
-    'tokens': fields.Integer,
-    'keywords': fields.List(fields.String),
-    'index_node_id': fields.String,
-    'index_node_hash': fields.String,
-    'hit_count': fields.Integer,
-    'enabled': fields.Boolean,
-    'disabled_at': TimestampField,
-    'disabled_by': fields.String,
-    'status': fields.String,
-    'created_by': fields.String,
-    'created_at': TimestampField,
-    'indexing_at': TimestampField,
-    'completed_at': TimestampField,
-    'error': fields.String,
-    'stopped_at': TimestampField,
-    'document': fields.Nested(document_fields),
-}
-
-hit_testing_record_fields = {
-    'segment': fields.Nested(segment_fields),
-    'score': fields.Float,
-    'tsne_position': fields.Raw
-}
-
 
 class HitTestingApi(Resource):
 

api/controllers/console/explore/conversation.py

@@ -7,26 +7,12 @@ from werkzeug.exceptions import NotFound
 from controllers.console import api
 from controllers.console.explore.error import NotChatAppError
 from controllers.console.explore.wraps import InstalledAppResource
+from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
 from libs.helper import TimestampField, uuid_value
 from services.conversation_service import ConversationService
 from services.errors.conversation import LastConversationNotExistsError, ConversationNotExistsError
 from services.web_conversation_service import WebConversationService
 
-conversation_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'inputs': fields.Raw,
-    'status': fields.String,
-    'introduction': fields.String,
-    'created_at': TimestampField
-}
-
-conversation_infinite_scroll_pagination_fields = {
-    'limit': fields.Integer,
-    'has_more': fields.Boolean,
-    'data': fields.List(fields.Nested(conversation_fields))
-}
-
 
 class ConversationListApi(InstalledAppResource):
 
@@ -76,7 +62,7 @@ class ConversationApi(InstalledAppResource):
 
 class ConversationRenameApi(InstalledAppResource):
 
-    @marshal_with(conversation_fields)
+    @marshal_with(simple_conversation_fields)
     def post(self, installed_app, c_id):
         app_model = installed_app.app
         if app_model.mode != 'chat':

api/controllers/console/explore/installed_app.py

@@ -11,32 +11,11 @@ from controllers.console import api
 from controllers.console.explore.wraps import InstalledAppResource
 from controllers.console.wraps import account_initialization_required
 from extensions.ext_database import db
+from fields.installed_app_fields import installed_app_list_fields
 from libs.helper import TimestampField
 from models.model import App, InstalledApp, RecommendedApp
 from services.account_service import TenantService
 
-app_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'mode': fields.String,
-    'icon': fields.String,
-    'icon_background': fields.String
-}
-
-installed_app_fields = {
-    'id': fields.String,
-    'app': fields.Nested(app_fields),
-    'app_owner_tenant_id': fields.String,
-    'is_pinned': fields.Boolean,
-    'last_used_at': TimestampField,
-    'editable': fields.Boolean,
-    'uninstallable': fields.Boolean,
-}
-
-installed_app_list_fields = {
-    'installed_apps': fields.List(fields.Nested(installed_app_fields))
-}
-
 
 class InstalledAppsListApi(Resource):
     @login_required

api/controllers/console/explore/message.py

@@ -17,6 +17,7 @@ from controllers.console.explore.error import NotCompletionAppError, AppSuggeste
 from controllers.console.explore.wraps import InstalledAppResource
 from core.model_providers.error import LLMRateLimitError, LLMBadRequestError, LLMAuthorizationError, LLMAPIConnectionError, \
     ProviderTokenNotInitError, LLMAPIUnavailableError, QuotaExceededError, ModelCurrentlyNotSupportError
+from fields.message_fields import message_infinite_scroll_pagination_fields
 from libs.helper import uuid_value, TimestampField
 from services.completion_service import CompletionService
 from services.errors.app import MoreLikeThisDisabledError
@@ -26,45 +27,6 @@ from services.message_service import MessageService
 
 
 class MessageListApi(InstalledAppResource):
-    feedback_fields = {
-        'rating': fields.String
-    }
-
-    retriever_resource_fields = {
-        'id': fields.String,
-        'message_id': fields.String,
-        'position': fields.Integer,
-        'dataset_id': fields.String,
-        'dataset_name': fields.String,
-        'document_id': fields.String,
-        'document_name': fields.String,
-        'data_source_type': fields.String,
-        'segment_id': fields.String,
-        'score': fields.Float,
-        'hit_count': fields.Integer,
-        'word_count': fields.Integer,
-        'segment_position': fields.Integer,
-        'index_node_hash': fields.String,
-        'content': fields.String,
-        'created_at': TimestampField
-    }
-
-    message_fields = {
-        'id': fields.String,
-        'conversation_id': fields.String,
-        'inputs': fields.Raw,
-        'query': fields.String,
-        'answer': fields.String,
-        'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
-        'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
-        'created_at': TimestampField
-    }
-
-    message_infinite_scroll_pagination_fields = {
-        'limit': fields.Integer,
-        'has_more': fields.Boolean,
-        'data': fields.List(fields.Nested(message_fields))
-    }
 
     @marshal_with(message_infinite_scroll_pagination_fields)
     def get(self, installed_app):

api/controllers/console/setup.py

@@ -1,7 +1,6 @@
 # -*- coding:utf-8 -*-
 from functools import wraps
 
-import flask_login
 from flask import request, current_app
 from flask_restful import Resource, reqparse
 
@@ -58,9 +57,6 @@ class SetupApi(Resource):
         )
 
         setup()
-
-        # Login
-        flask_login.login_user(account)
         AccountService.update_last_login(account, request)
 
         return {'result': 'success'}, 201

api/controllers/console/universal_chat/conversation.py

@@ -6,31 +6,17 @@ from werkzeug.exceptions import NotFound
 
 from controllers.console import api
 from controllers.console.universal_chat.wraps import UniversalChatResource
+from fields.conversation_fields import conversation_with_model_config_infinite_scroll_pagination_fields, \
+    conversation_with_model_config_fields
 from libs.helper import TimestampField, uuid_value
 from services.conversation_service import ConversationService
 from services.errors.conversation import LastConversationNotExistsError, ConversationNotExistsError
 from services.web_conversation_service import WebConversationService
 
-conversation_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'inputs': fields.Raw,
-    'status': fields.String,
-    'introduction': fields.String,
-    'created_at': TimestampField,
-    'model_config': fields.Raw,
-}
-
-conversation_infinite_scroll_pagination_fields = {
-    'limit': fields.Integer,
-    'has_more': fields.Boolean,
-    'data': fields.List(fields.Nested(conversation_fields))
-}
-
 
 class UniversalChatConversationListApi(UniversalChatResource):
 
-    @marshal_with(conversation_infinite_scroll_pagination_fields)
+    @marshal_with(conversation_with_model_config_infinite_scroll_pagination_fields)
     def get(self, universal_app):
         app_model = universal_app
 
@@ -73,7 +59,7 @@ class UniversalChatConversationApi(UniversalChatResource):
 
 class UniversalChatConversationRenameApi(UniversalChatResource):
 
-    @marshal_with(conversation_fields)
+    @marshal_with(conversation_with_model_config_fields)
     def post(self, universal_app, c_id):
         app_model = universal_app
         conversation_id = str(c_id)

api/controllers/service_api/__init__.py

@@ -9,4 +9,4 @@ api = ExternalApi(bp)
 
 from .app import completion, app, conversation, message, audio
 
-from .dataset import document
+from .dataset import document, segment, dataset

api/controllers/service_api/app/conversation.py

@@ -8,25 +8,11 @@ from controllers.service_api import api
 from controllers.service_api.app import create_or_update_end_user_for_user_id
 from controllers.service_api.app.error import NotChatAppError
 from controllers.service_api.wraps import AppApiResource
+from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
 from libs.helper import TimestampField, uuid_value
 import services
 from services.conversation_service import ConversationService
 
-conversation_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'inputs': fields.Raw,
-    'status': fields.String,
-    'introduction': fields.String,
-    'created_at': TimestampField
-}
-
-conversation_infinite_scroll_pagination_fields = {
-    'limit': fields.Integer,
-    'has_more': fields.Boolean,
-    'data': fields.List(fields.Nested(conversation_fields))
-}
-
 
 class ConversationApi(AppApiResource):
 
@@ -50,7 +36,7 @@ class ConversationApi(AppApiResource):
             raise NotFound("Last Conversation Not Exists.")
 
 class ConversationDetailApi(AppApiResource):
-    @marshal_with(conversation_fields)
+    @marshal_with(simple_conversation_fields)
     def delete(self, app_model, end_user, c_id):
         if app_model.mode != 'chat':
             raise NotChatAppError()
@@ -70,7 +56,7 @@ class ConversationDetailApi(AppApiResource):
 
 class ConversationRenameApi(AppApiResource):
 
-    @marshal_with(conversation_fields)
+    @marshal_with(simple_conversation_fields)
     def post(self, app_model, end_user, c_id):
         if app_model.mode != 'chat':
             raise NotChatAppError()

api/controllers/service_api/dataset/dataset.py

@@ -0,0 +1,84 @@
+from flask import request
+from flask_restful import reqparse, marshal
+import services.dataset_service
+from controllers.service_api import api
+from controllers.service_api.dataset.error import DatasetNameDuplicateError
+from controllers.service_api.wraps import DatasetApiResource
+from core.login.login import current_user
+from core.model_providers.models.entity.model_params import ModelType
+from extensions.ext_database import db
+from fields.dataset_fields import dataset_detail_fields
+from models.account import Account, TenantAccountJoin
+from models.dataset import Dataset
+from services.dataset_service import DatasetService
+from services.provider_service import ProviderService
+
+
+def _validate_name(name):
+    if not name or len(name) < 1 or len(name) > 40:
+        raise ValueError('Name must be between 1 to 40 characters.')
+    return name
+
+
+class DatasetApi(DatasetApiResource):
+    """Resource for get datasets."""
+
+    def get(self, tenant_id):
+        page = request.args.get('page', default=1, type=int)
+        limit = request.args.get('limit', default=20, type=int)
+        provider = request.args.get('provider', default="vendor")
+        datasets, total = DatasetService.get_datasets(page, limit, provider,
+                                                      tenant_id, current_user)
+        # check embedding setting
+        provider_service = ProviderService()
+        valid_model_list = provider_service.get_valid_model_list(current_user.current_tenant_id,
+                                                                 ModelType.EMBEDDINGS.value)
+        model_names = []
+        for valid_model in valid_model_list:
+            model_names.append(f"{valid_model['model_name']}:{valid_model['model_provider']['provider_name']}")
+        data = marshal(datasets, dataset_detail_fields)
+        for item in data:
+            if item['indexing_technique'] == 'high_quality':
+                item_model = f"{item['embedding_model']}:{item['embedding_model_provider']}"
+                if item_model in model_names:
+                    item['embedding_available'] = True
+                else:
+                    item['embedding_available'] = False
+            else:
+                item['embedding_available'] = True
+        response = {
+            'data': data,
+            'has_more': len(datasets) == limit,
+            'limit': limit,
+            'total': total,
+            'page': page
+        }
+        return response, 200
+
+    """Resource for datasets."""
+
+    def post(self, tenant_id):
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', nullable=False, required=True,
+                            help='type is required. Name must be between 1 to 40 characters.',
+                            type=_validate_name)
+        parser.add_argument('indexing_technique', type=str, location='json',
+                            choices=('high_quality', 'economy'),
+                            help='Invalid indexing technique.')
+        args = parser.parse_args()
+
+        try:
+            dataset = DatasetService.create_empty_dataset(
+                tenant_id=tenant_id,
+                name=args['name'],
+                indexing_technique=args['indexing_technique'],
+                account=current_user
+            )
+        except services.errors.dataset.DatasetNameDuplicateError:
+            raise DatasetNameDuplicateError()
+
+        return marshal(dataset, dataset_detail_fields), 200
+
+
+api.add_resource(DatasetApi, '/datasets')
+

api/controllers/service_api/dataset/document.py

@@ -1,114 +1,291 @@
 import datetime
+import json
 import uuid
 
-from flask import current_app
-from flask_restful import reqparse
+from flask import current_app, request
+from flask_restful import reqparse, marshal
+from sqlalchemy import desc
 from werkzeug.exceptions import NotFound
 
 import services.dataset_service
 from controllers.service_api import api
 from controllers.service_api.app.error import ProviderNotInitializeError
 from controllers.service_api.dataset.error import ArchivedDocumentImmutableError, DocumentIndexingError, \
-    DatasetNotInitedError
+    NoFileUploadedError, TooManyFilesError
 from controllers.service_api.wraps import DatasetApiResource
+from core.login.login import current_user
 from core.model_providers.error import ProviderTokenNotInitError
 from extensions.ext_database import db
 from extensions.ext_storage import storage
+from fields.document_fields import document_fields, document_status_fields
+from models.dataset import Dataset, Document, DocumentSegment
 from models.model import UploadFile
 from services.dataset_service import DocumentService
+from services.file_service import FileService
 
 
-class DocumentListApi(DatasetApiResource):
+class DocumentAddByTextApi(DatasetApiResource):
     """Resource for documents."""
 
-    def post(self, dataset):
-        """Create document."""
+    def post(self, tenant_id, dataset_id):
+        """Create document by text."""
         parser = reqparse.RequestParser()
         parser.add_argument('name', type=str, required=True, nullable=False, location='json')
         parser.add_argument('text', type=str, required=True, nullable=False, location='json')
-        parser.add_argument('doc_type', type=str, location='json')
-        parser.add_argument('doc_metadata', type=dict, location='json')
+        parser.add_argument('process_rule', type=dict, required=False, nullable=True, location='json')
+        parser.add_argument('original_document_id', type=str, required=False, location='json')
+        parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
+        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False,
+                            location='json')
+        parser.add_argument('indexing_technique', type=str, choices=Dataset.INDEXING_TECHNIQUE_LIST, nullable=False,
+                            location='json')
         args = parser.parse_args()
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        dataset = db.session.query(Dataset).filter(
+            Dataset.tenant_id == tenant_id,
+            Dataset.id == dataset_id
+        ).first()
 
-        if not dataset.indexing_technique:
-            raise DatasetNotInitedError("Dataset indexing technique must be set.")
+        if not dataset:
+            raise ValueError('Dataset is not exist.')
 
-        doc_type = args.get('doc_type')
-        doc_metadata = args.get('doc_metadata')
+        if not dataset.indexing_technique and not args['indexing_technique']:
+            raise ValueError('indexing_technique is required.')
 
-        if doc_type and doc_type not in DocumentService.DOCUMENT_METADATA_SCHEMA:
-            raise ValueError('Invalid doc_type.')
-
-        # user uuid as file name
-        file_uuid = str(uuid.uuid4())
-        file_key = 'upload_files/' + dataset.tenant_id + '/' + file_uuid + '.txt'
-
-        # save file to storage
-        storage.save(file_key, args.get('text'))
-
-        # save file to db
-        config = current_app.config
-        upload_file = UploadFile(
-            tenant_id=dataset.tenant_id,
-            storage_type=config['STORAGE_TYPE'],
-            key=file_key,
-            name=args.get('name') + '.txt',
-            size=len(args.get('text')),
-            extension='txt',
-            mime_type='text/plain',
-            created_by=dataset.created_by,
-            created_at=datetime.datetime.utcnow(),
-            used=True,
-            used_by=dataset.created_by,
-            used_at=datetime.datetime.utcnow()
-        )
-
-        db.session.add(upload_file)
-        db.session.commit()
-
-        document_data = {
-            'data_source': {
-                'type': 'upload_file',
-                'info': [
-                    {
-                        'upload_file_id': upload_file.id
-                    }
-                ]
+        upload_file = FileService.upload_text(args.get('text'), args.get('name'))
+        data_source = {
+            'type': 'upload_file',
+            'info_list': {
+                'data_source_type': 'upload_file',
+                'file_info_list': {
+                    'file_ids': [upload_file.id]
+                }
             }
         }
+        args['data_source'] = data_source
+        # validate args
+        DocumentService.document_create_args_validate(args)
 
         try:
             documents, batch = DocumentService.save_document_with_dataset_id(
                 dataset=dataset,
-                document_data=document_data,
-                account=dataset.created_by_account,
-                dataset_process_rule=dataset.latest_process_rule,
+                document_data=args,
+                account=current_user,
+                dataset_process_rule=dataset.latest_process_rule if 'process_rule' not in args else None,
                 created_from='api'
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
         document = documents[0]
-        if doc_type and doc_metadata:
-            metadata_schema = DocumentService.DOCUMENT_METADATA_SCHEMA[doc_type]
 
-            document.doc_metadata = {}
-
-            for key, value_type in metadata_schema.items():
-                value = doc_metadata.get(key)
-                if value is not None and isinstance(value, value_type):
-                    document.doc_metadata[key] = value
-
-            document.doc_type = doc_type
-            document.updated_at = datetime.datetime.utcnow()
-            db.session.commit()
-
-        return {'id': document.id}
+        documents_and_batch_fields = {
+            'document': marshal(document, document_fields),
+            'batch': batch
+        }
+        return documents_and_batch_fields, 200
 
 
-class DocumentApi(DatasetApiResource):
-    def delete(self, dataset, document_id):
+class DocumentUpdateByTextApi(DatasetApiResource):
+    """Resource for update documents."""
+
+    def post(self, tenant_id, dataset_id, document_id):
+        """Update document by text."""
+        parser = reqparse.RequestParser()
+        parser.add_argument('name', type=str, required=False, nullable=True, location='json')
+        parser.add_argument('text', type=str, required=False, nullable=True, location='json')
+        parser.add_argument('process_rule', type=dict, required=False, nullable=True, location='json')
+        parser.add_argument('doc_form', type=str, default='text_model', required=False, nullable=False, location='json')
+        parser.add_argument('doc_language', type=str, default='English', required=False, nullable=False,
+                            location='json')
+        args = parser.parse_args()
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        dataset = db.session.query(Dataset).filter(
+            Dataset.tenant_id == tenant_id,
+            Dataset.id == dataset_id
+        ).first()
+
+        if not dataset:
+            raise ValueError('Dataset is not exist.')
+
+        if args['text']:
+            upload_file = FileService.upload_text(args.get('text'), args.get('name'))
+            data_source = {
+                'type': 'upload_file',
+                'info_list': {
+                    'data_source_type': 'upload_file',
+                    'file_info_list': {
+                        'file_ids': [upload_file.id]
+                    }
+                }
+            }
+            args['data_source'] = data_source
+        # validate args
+        args['original_document_id'] = str(document_id)
+        DocumentService.document_create_args_validate(args)
+
+        try:
+            documents, batch = DocumentService.save_document_with_dataset_id(
+                dataset=dataset,
+                document_data=args,
+                account=current_user,
+                dataset_process_rule=dataset.latest_process_rule if 'process_rule' not in args else None,
+                created_from='api'
+            )
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        document = documents[0]
+
+        documents_and_batch_fields = {
+            'document': marshal(document, document_fields),
+            'batch': batch
+        }
+        return documents_and_batch_fields, 200
+
+
+class DocumentAddByFileApi(DatasetApiResource):
+    """Resource for documents."""
+    def post(self, tenant_id, dataset_id):
+        """Create document by upload file."""
+        args = {}
+        if 'data' in request.form:
+            args = json.loads(request.form['data'])
+        if 'doc_form' not in args:
+            args['doc_form'] = 'text_model'
+        if 'doc_language' not in args:
+            args['doc_language'] = 'English'
+        # get dataset info
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        dataset = db.session.query(Dataset).filter(
+            Dataset.tenant_id == tenant_id,
+            Dataset.id == dataset_id
+        ).first()
+
+        if not dataset:
+            raise ValueError('Dataset is not exist.')
+        if not dataset.indexing_technique and not args['indexing_technique']:
+            raise ValueError('indexing_technique is required.')
+
+        # save file info
+        file = request.files['file']
+        # check file
+        if 'file' not in request.files:
+            raise NoFileUploadedError()
+
+        if len(request.files) > 1:
+            raise TooManyFilesError()
+
+        upload_file = FileService.upload_file(file)
+        data_source = {
+            'type': 'upload_file',
+            'info_list': {
+                'file_info_list': {
+                    'file_ids': [upload_file.id]
+                }
+            }
+        }
+        args['data_source'] = data_source
+        # validate args
+        DocumentService.document_create_args_validate(args)
+
+        try:
+            documents, batch = DocumentService.save_document_with_dataset_id(
+                dataset=dataset,
+                document_data=args,
+                account=dataset.created_by_account,
+                dataset_process_rule=dataset.latest_process_rule if 'process_rule' not in args else None,
+                created_from='api'
+            )
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        document = documents[0]
+        documents_and_batch_fields = {
+            'document': marshal(document, document_fields),
+            'batch': batch
+        }
+        return documents_and_batch_fields, 200
+
+
+class DocumentUpdateByFileApi(DatasetApiResource):
+    """Resource for update documents."""
+
+    def post(self, tenant_id, dataset_id, document_id):
+        """Update document by upload file."""
+        args = {}
+        if 'data' in request.form:
+            args = json.loads(request.form['data'])
+        if 'doc_form' not in args:
+            args['doc_form'] = 'text_model'
+        if 'doc_language' not in args:
+            args['doc_language'] = 'English'
+
+        # get dataset info
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        dataset = db.session.query(Dataset).filter(
+            Dataset.tenant_id == tenant_id,
+            Dataset.id == dataset_id
+        ).first()
+
+        if not dataset:
+            raise ValueError('Dataset is not exist.')
+        if 'file' in request.files:
+            # save file info
+            file = request.files['file']
+
+
+            if len(request.files) > 1:
+                raise TooManyFilesError()
+
+            upload_file = FileService.upload_file(file)
+            data_source = {
+                'type': 'upload_file',
+                'info_list': {
+                    'file_info_list': {
+                        'file_ids': [upload_file.id]
+                    }
+                }
+            }
+            args['data_source'] = data_source
+        # validate args
+        args['original_document_id'] = str(document_id)
+        DocumentService.document_create_args_validate(args)
+
+        try:
+            documents, batch = DocumentService.save_document_with_dataset_id(
+                dataset=dataset,
+                document_data=args,
+                account=dataset.created_by_account,
+                dataset_process_rule=dataset.latest_process_rule if 'process_rule' not in args else None,
+                created_from='api'
+            )
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        document = documents[0]
+        documents_and_batch_fields = {
+            'document': marshal(document, document_fields),
+            'batch': batch
+        }
+        return documents_and_batch_fields, 200
+
+
+class DocumentDeleteApi(DatasetApiResource):
+    def delete(self, tenant_id, dataset_id, document_id):
         """Delete document."""
         document_id = str(document_id)
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+
+        # get dataset info
+        dataset = db.session.query(Dataset).filter(
+            Dataset.tenant_id == tenant_id,
+            Dataset.id == dataset_id
+        ).first()
+
+        if not dataset:
+            raise ValueError('Dataset is not exist.')
 
         document = DocumentService.get_document(dataset.id, document_id)
 
@@ -126,8 +303,85 @@ class DocumentApi(DatasetApiResource):
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError('Cannot delete document during indexing.')
 
-        return {'result': 'success'}, 204
+        return {'result': 'success'}, 200
 
 
-api.add_resource(DocumentListApi, '/documents')
-api.add_resource(DocumentApi, '/documents/<uuid:document_id>')
+class DocumentListApi(DatasetApiResource):
+    def get(self, tenant_id, dataset_id):
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        page = request.args.get('page', default=1, type=int)
+        limit = request.args.get('limit', default=20, type=int)
+        search = request.args.get('keyword', default=None, type=str)
+        dataset = db.session.query(Dataset).filter(
+            Dataset.tenant_id == tenant_id,
+            Dataset.id == dataset_id
+        ).first()
+        if not dataset:
+            raise NotFound('Dataset not found.')
+
+        query = Document.query.filter_by(
+            dataset_id=str(dataset_id), tenant_id=tenant_id)
+
+        if search:
+            search = f'%{search}%'
+            query = query.filter(Document.name.like(search))
+
+        query = query.order_by(desc(Document.created_at))
+
+        paginated_documents = query.paginate(
+            page=page, per_page=limit, max_per_page=100, error_out=False)
+        documents = paginated_documents.items
+
+        response = {
+            'data': marshal(documents, document_fields),
+            'has_more': len(documents) == limit,
+            'limit': limit,
+            'total': paginated_documents.total,
+            'page': page
+        }
+
+        return response
+
+
+class DocumentIndexingStatusApi(DatasetApiResource):
+    def get(self, tenant_id, dataset_id, batch):
+        dataset_id = str(dataset_id)
+        batch = str(batch)
+        tenant_id = str(tenant_id)
+        # get dataset
+        dataset = db.session.query(Dataset).filter(
+            Dataset.tenant_id == tenant_id,
+            Dataset.id == dataset_id
+        ).first()
+        if not dataset:
+            raise NotFound('Dataset not found.')
+        # get documents
+        documents = DocumentService.get_batch_documents(dataset_id, batch)
+        if not documents:
+            raise NotFound('Documents not found.')
+        documents_status = []
+        for document in documents:
+            completed_segments = DocumentSegment.query.filter(DocumentSegment.completed_at.isnot(None),
+                                                              DocumentSegment.document_id == str(document.id),
+                                                              DocumentSegment.status != 're_segment').count()
+            total_segments = DocumentSegment.query.filter(DocumentSegment.document_id == str(document.id),
+                                                          DocumentSegment.status != 're_segment').count()
+            document.completed_segments = completed_segments
+            document.total_segments = total_segments
+            if document.is_paused:
+                document.indexing_status = 'paused'
+            documents_status.append(marshal(document, document_status_fields))
+        data = {
+            'data': documents_status
+        }
+        return data
+
+
+api.add_resource(DocumentAddByTextApi, '/datasets/<uuid:dataset_id>/document/create_by_text')
+api.add_resource(DocumentAddByFileApi, '/datasets/<uuid:dataset_id>/document/create_by_file')
+api.add_resource(DocumentUpdateByTextApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_text')
+api.add_resource(DocumentUpdateByFileApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file')
+api.add_resource(DocumentDeleteApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>')
+api.add_resource(DocumentListApi, '/datasets/<uuid:dataset_id>/documents')
+api.add_resource(DocumentIndexingStatusApi, '/datasets/<uuid:dataset_id>/documents/<string:batch>/indexing-status')

api/controllers/service_api/dataset/error.py

@@ -1,20 +1,73 @@
-# -*- coding:utf-8 -*-
 from libs.exception import BaseHTTPException
 
 
+class NoFileUploadedError(BaseHTTPException):
+    error_code = 'no_file_uploaded'
+    description = "Please upload your file."
+    code = 400
+
+
+class TooManyFilesError(BaseHTTPException):
+    error_code = 'too_many_files'
+    description = "Only one file is allowed."
+    code = 400
+
+
+class FileTooLargeError(BaseHTTPException):
+    error_code = 'file_too_large'
+    description = "File size exceeded. {message}"
+    code = 413
+
+
+class UnsupportedFileTypeError(BaseHTTPException):
+    error_code = 'unsupported_file_type'
+    description = "File type not allowed."
+    code = 415
+
+
+class HighQualityDatasetOnlyError(BaseHTTPException):
+    error_code = 'high_quality_dataset_only'
+    description = "Current operation only supports 'high-quality' datasets."
+    code = 400
+
+
+class DatasetNotInitializedError(BaseHTTPException):
+    error_code = 'dataset_not_initialized'
+    description = "The dataset is still being initialized or indexing. Please wait a moment."
+    code = 400
+
+
 class ArchivedDocumentImmutableError(BaseHTTPException):
     error_code = 'archived_document_immutable'
-    description = "Cannot operate when document was archived."
+    description = "The archived document is not editable."
     code = 403
 
 
+class DatasetNameDuplicateError(BaseHTTPException):
+    error_code = 'dataset_name_duplicate'
+    description = "The dataset name already exists. Please modify your dataset name."
+    code = 409
+
+
+class InvalidActionError(BaseHTTPException):
+    error_code = 'invalid_action'
+    description = "Invalid action."
+    code = 400
+
+
+class DocumentAlreadyFinishedError(BaseHTTPException):
+    error_code = 'document_already_finished'
+    description = "The document has been processed. Please refresh the page or go to the document details."
+    code = 400
+
+
 class DocumentIndexingError(BaseHTTPException):
     error_code = 'document_indexing'
-    description = "Cannot operate document during indexing."
-    code = 403
+    description = "The document is being processed and cannot be edited."
+    code = 400
 
 
-class DatasetNotInitedError(BaseHTTPException):
-    error_code = 'dataset_not_inited'
-    description = "The dataset is still being initialized or indexing. Please wait a moment."
-    code = 403
+class InvalidMetadataError(BaseHTTPException):
+    error_code = 'invalid_metadata'
+    description = "The metadata content is incorrect. Please check and verify."
+    code = 400

api/controllers/service_api/dataset/segment.py

@@ -0,0 +1,59 @@
+from flask_login import current_user
+from flask_restful import reqparse, marshal
+from werkzeug.exceptions import NotFound
+
+from controllers.service_api import api
+from controllers.service_api.app.error import ProviderNotInitializeError
+from controllers.service_api.wraps import DatasetApiResource
+from core.model_providers.error import ProviderTokenNotInitError, LLMBadRequestError
+from core.model_providers.model_factory import ModelFactory
+from extensions.ext_database import db
+from fields.segment_fields import segment_fields
+from models.dataset import Dataset
+from services.dataset_service import DocumentService, SegmentService
+
+
+class SegmentApi(DatasetApiResource):
+    """Resource for segments."""
+    def post(self, tenant_id, dataset_id, document_id):
+        """Create single segment."""
+        # check dataset
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+        dataset = db.session.query(Dataset).filter(
+            Dataset.tenant_id == tenant_id,
+            Dataset.id == dataset_id
+        ).first()
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset.id, document_id)
+        if not document:
+            raise NotFound('Document not found.')
+        # check embedding model setting
+        if dataset.indexing_technique == 'high_quality':
+            try:
+                ModelFactory.get_embedding_model(
+                    tenant_id=current_user.current_tenant_id,
+                    model_provider_name=dataset.embedding_model_provider,
+                    model_name=dataset.embedding_model
+                )
+            except LLMBadRequestError:
+                raise ProviderNotInitializeError(
+                    f"No Embedding Model available. Please configure a valid provider "
+                    f"in the Settings -> Model Provider.")
+            except ProviderTokenNotInitError as ex:
+                raise ProviderNotInitializeError(ex.description)
+        # validate args
+        parser = reqparse.RequestParser()
+        parser.add_argument('segments', type=list, required=False, nullable=True, location='json')
+        args = parser.parse_args()
+        for args_item in args['segments']:
+            SegmentService.segment_create_args_validate(args_item, document)
+        segments = SegmentService.multi_create_segment(args['segments'], document, dataset)
+        return {
+            'data': marshal(segments, segment_fields),
+            'doc_form': document.doc_form
+        }, 200
+
+
+api.add_resource(SegmentApi, '/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments')

api/controllers/service_api/wraps.py

@@ -2,11 +2,14 @@
 from datetime import datetime
 from functools import wraps
 
-from flask import request
+from flask import request, current_app
+from flask_login import user_logged_in
 from flask_restful import Resource
 from werkzeug.exceptions import NotFound, Unauthorized
 
+from core.login.login import _get_user
 from extensions.ext_database import db
+from models.account import Tenant, TenantAccountJoin, Account
 from models.dataset import Dataset
 from models.model import ApiToken, App
 
@@ -43,12 +46,24 @@ def validate_dataset_token(view=None):
         @wraps(view)
         def decorated(*args, **kwargs):
             api_token = validate_and_get_api_token('dataset')
-
-            dataset = db.session.query(Dataset).filter(Dataset.id == api_token.dataset_id).first()
-            if not dataset:
-                raise NotFound()
-
-            return view(dataset, *args, **kwargs)
+            tenant_account_join = db.session.query(Tenant, TenantAccountJoin) \
+                .filter(Tenant.id == api_token.tenant_id) \
+                .filter(TenantAccountJoin.tenant_id == Tenant.id) \
+                .filter(TenantAccountJoin.role == 'owner') \
+                .one_or_none()
+            if tenant_account_join:
+                tenant, ta = tenant_account_join
+                account = Account.query.filter_by(id=ta.account_id).first()
+                # Login admin
+                if account:
+                    account.current_tenant = tenant
+                    current_app.login_manager._update_request_context_with_user(account)
+                    user_logged_in.send(current_app._get_current_object(), user=_get_user())
+                else:
+                    raise Unauthorized("Tenant owner account is not exist.")
+            else:
+                raise Unauthorized("Tenant is not exist.")
+            return view(api_token.tenant_id, *args, **kwargs)
         return decorated
 
     if view:

api/controllers/web/conversation.py

@@ -6,26 +6,12 @@ from werkzeug.exceptions import NotFound
 from controllers.web import api
 from controllers.web.error import NotChatAppError
 from controllers.web.wraps import WebApiResource
+from fields.conversation_fields import conversation_infinite_scroll_pagination_fields, simple_conversation_fields
 from libs.helper import TimestampField, uuid_value
 from services.conversation_service import ConversationService
 from services.errors.conversation import LastConversationNotExistsError, ConversationNotExistsError
 from services.web_conversation_service import WebConversationService
 
-conversation_fields = {
-    'id': fields.String,
-    'name': fields.String,
-    'inputs': fields.Raw,
-    'status': fields.String,
-    'introduction': fields.String,
-    'created_at': TimestampField
-}
-
-conversation_infinite_scroll_pagination_fields = {
-    'limit': fields.Integer,
-    'has_more': fields.Boolean,
-    'data': fields.List(fields.Nested(conversation_fields))
-}
-
 
 class ConversationListApi(WebApiResource):
 
@@ -73,7 +59,7 @@ class ConversationApi(WebApiResource):
 
 class ConversationRenameApi(WebApiResource):
 
-    @marshal_with(conversation_fields)
+    @marshal_with(simple_conversation_fields)
     def post(self, app_model, end_user, c_id):
         if app_model.mode != 'chat':
             raise NotChatAppError()

api/core/completion.py

@@ -108,12 +108,14 @@ class Completion:
                 retriever_from=retriever_from
             )
 
+            query_for_agent = cls.get_query_for_agent(app, app_model_config, query, inputs)
+
             # run agent executor
             agent_execute_result = None
-            if agent_executor:
-                should_use_agent = agent_executor.should_use_agent(query)
+            if query_for_agent and agent_executor:
+                should_use_agent = agent_executor.should_use_agent(query_for_agent)
                 if should_use_agent:
-                    agent_execute_result = agent_executor.run(query)
+                    agent_execute_result = agent_executor.run(query_for_agent)
 
             # When no extra pre prompt is specified,
             # the output of the agent can be used directly as the main output content without calling LLM again
@@ -142,6 +144,13 @@ class Completion:
             logging.warning(f'ChunkedEncodingError: {e}')
             conversation_message_task.end()
             return
+        
+    @classmethod
+    def get_query_for_agent(cls, app: App, app_model_config: AppModelConfig, query: str, inputs: dict) -> str:
+        if app.mode != 'completion':
+            return query
+        
+        return inputs.get(app_model_config.dataset_query_variable, "")
 
     @classmethod
     def run_final_llm(cls, model_instance: BaseLLM, mode: str, app_model_config: AppModelConfig, query: str,

api/core/conversation_message_task.py

@@ -112,7 +112,7 @@ class ConversationMessageTask:
             )
 
             db.session.add(self.conversation)
-            db.session.flush()
+            db.session.commit()
 
         self.message = Message(
             app_id=self.app_model_config.app_id,
@@ -140,7 +140,7 @@ class ConversationMessageTask:
         )
 
         db.session.add(self.message)
-        db.session.flush()
+        db.session.commit()
 
     def append_message_text(self, text: str):
         if text is not None:
@@ -191,12 +191,13 @@ class ConversationMessageTask:
         )
 
         db.session.add(message_chain)
-        db.session.flush()
+        db.session.commit()
 
         return message_chain
 
     def on_chain_end(self, message_chain: MessageChain, chain_result: ChainResult):
         message_chain.output = json.dumps(chain_result.completion)
+        db.session.commit()
 
         self._pub_handler.pub_chain(message_chain)
 
@@ -217,7 +218,7 @@ class ConversationMessageTask:
         )
 
         db.session.add(message_agent_thought)
-        db.session.flush()
+        db.session.commit()
 
         self._pub_handler.pub_agent_thought(message_agent_thought)
 
@@ -249,7 +250,7 @@ class ConversationMessageTask:
         message_agent_thought.tokens = agent_loop.prompt_tokens + agent_loop.completion_tokens
         message_agent_thought.total_price = loop_total_price
         message_agent_thought.currency = agent_model_instance.get_currency()
-        db.session.flush()
+        db.session.commit()
 
     def on_dataset_query_end(self, dataset_query_obj: DatasetQueryObj):
         dataset_query = DatasetQuery(
@@ -262,6 +263,7 @@ class ConversationMessageTask:
         )
 
         db.session.add(dataset_query)
+        db.session.commit()
 
     def on_dataset_query_finish(self, resource: List):
         if resource and len(resource) > 0:
@@ -285,7 +287,7 @@ class ConversationMessageTask:
                     created_by=self.user.id
                 )
                 db.session.add(dataset_retriever_resource)
-                db.session.flush()
+                db.session.commit()
             self.retriever_resource = resource
 
     def message_end(self):

api/core/data_loader/loader/notion.py

@@ -16,6 +16,7 @@ logger = logging.getLogger(__name__)
 BLOCK_CHILD_URL_TMPL = "https://api.notion.com/v1/blocks/{block_id}/children"
 DATABASE_URL_TMPL = "https://api.notion.com/v1/databases/{database_id}/query"
 SEARCH_URL = "https://api.notion.com/v1/search"
+
 RETRIEVE_PAGE_URL_TMPL = "https://api.notion.com/v1/pages/{page_id}"
 RETRIEVE_DATABASE_URL_TMPL = "https://api.notion.com/v1/databases/{database_id}"
 HEADING_TYPE = ['heading_1', 'heading_2', 'heading_3']

api/core/index/keyword_table_index/keyword_table_index.py

@@ -246,11 +246,28 @@ class KeywordTableIndex(BaseIndex):
         keyword_table = self._add_text_to_keyword_table(keyword_table, node_id, keywords)
         self._save_dataset_keyword_table(keyword_table)
 
+    def multi_create_segment_keywords(self, pre_segment_data_list: list):
+        keyword_table_handler = JiebaKeywordTableHandler()
+        keyword_table = self._get_dataset_keyword_table()
+        for pre_segment_data in pre_segment_data_list:
+            segment = pre_segment_data['segment']
+            if pre_segment_data['keywords']:
+                segment.keywords = pre_segment_data['keywords']
+                keyword_table = self._add_text_to_keyword_table(keyword_table, segment.index_node_id,
+                                                                pre_segment_data['keywords'])
+            else:
+                keywords = keyword_table_handler.extract_keywords(segment.content,
+                                                                  self._config.max_keywords_per_chunk)
+                segment.keywords = list(keywords)
+                keyword_table = self._add_text_to_keyword_table(keyword_table, segment.index_node_id, list(keywords))
+        self._save_dataset_keyword_table(keyword_table)
+
     def update_segment_keywords_index(self, node_id: str, keywords: List[str]):
         keyword_table = self._get_dataset_keyword_table()
         keyword_table = self._add_text_to_keyword_table(keyword_table, node_id, keywords)
         self._save_dataset_keyword_table(keyword_table)
 
+
 class KeywordTableRetriever(BaseRetriever, BaseModel):
     index: KeywordTableIndex
     search_kwargs: dict = Field(default_factory=dict)

api/core/index/vector_index/base.py

@@ -113,8 +113,10 @@ class BaseVectorIndex(BaseIndex):
     def delete_by_group_id(self, group_id: str) -> None:
         vector_store = self._get_vector_store()
         vector_store = cast(self._get_vector_store_class(), vector_store)
-
-        vector_store.delete()
+        if self.dataset.collection_binding_id:
+            vector_store.delete_by_group_id(group_id)
+        else:
+            vector_store.delete()
 
     def delete(self) -> None:
         vector_store = self._get_vector_store()
@@ -283,7 +285,7 @@ class BaseVectorIndex(BaseIndex):
 
         if documents:
             try:
-                self.create_with_collection_name(documents, dataset_collection_binding.collection_name)
+                self.add_texts(documents)
             except Exception as e:
                 raise e
 

api/core/index/vector_index/qdrant.py

@@ -1390,70 +1390,12 @@ class Qdrant(VectorStore):
             path=path,
             **kwargs,
         )
-        try:
-            # Skip any validation in case of forced collection recreate.
-            if force_recreate:
-                raise ValueError
-
-            # Get the vector configuration of the existing collection and vector, if it
-            # was specified. If the old configuration does not match the current one,
-            # an exception is being thrown.
-            collection_info = client.get_collection(collection_name=collection_name)
-            current_vector_config = collection_info.config.params.vectors
-            if isinstance(current_vector_config, dict) and vector_name is not None:
-                if vector_name not in current_vector_config:
-                    raise QdrantException(
-                        f"Existing Qdrant collection {collection_name} does not "
-                        f"contain vector named {vector_name}. Did you mean one of the "
-                        f"existing vectors: {', '.join(current_vector_config.keys())}? "
-                        f"If you want to recreate the collection, set `force_recreate` "
-                        f"parameter to `True`."
-                    )
-                current_vector_config = current_vector_config.get(
-                    vector_name
-                )  # type: ignore[assignment]
-            elif isinstance(current_vector_config, dict) and vector_name is None:
-                raise QdrantException(
-                    f"Existing Qdrant collection {collection_name} uses named vectors. "
-                    f"If you want to reuse it, please set `vector_name` to any of the "
-                    f"existing named vectors: "
-                    f"{', '.join(current_vector_config.keys())}."  # noqa
-                    f"If you want to recreate the collection, set `force_recreate` "
-                    f"parameter to `True`."
-                )
-            elif (
-                not isinstance(current_vector_config, dict) and vector_name is not None
-            ):
-                raise QdrantException(
-                    f"Existing Qdrant collection {collection_name} doesn't use named "
-                    f"vectors. If you want to reuse it, please set `vector_name` to "
-                    f"`None`. If you want to recreate the collection, set "
-                    f"`force_recreate` parameter to `True`."
-                )
-
-            # Check if the vector configuration has the same dimensionality.
-            if current_vector_config.size != vector_size:  # type: ignore[union-attr]
-                raise QdrantException(
-                    f"Existing Qdrant collection is configured for vectors with "
-                    f"{current_vector_config.size} "  # type: ignore[union-attr]
-                    f"dimensions. Selected embeddings are {vector_size}-dimensional. "
-                    f"If you want to recreate the collection, set `force_recreate` "
-                    f"parameter to `True`."
-                )
-
-            current_distance_func = (
-                current_vector_config.distance.name.upper()  # type: ignore[union-attr]
-            )
-            if current_distance_func != distance_func:
-                raise QdrantException(
-                    f"Existing Qdrant collection is configured for "
-                    f"{current_vector_config.distance} "  # type: ignore[union-attr]
-                    f"similarity. Please set `distance_func` parameter to "
-                    f"`{distance_func}` if you want to reuse it. If you want to "
-                    f"recreate the collection, set `force_recreate` parameter to "
-                    f"`True`."
-                )
-        except (UnexpectedResponse, RpcError, ValueError):
+        all_collection_name = []
+        collections_response = client.get_collections()
+        collection_list = collections_response.collections
+        for collection in collection_list:
+            all_collection_name.append(collection.name)
+        if collection_name not in all_collection_name:
             vectors_config = rest.VectorParams(
                 size=vector_size,
                 distance=rest.Distance[distance_func],
@@ -1481,6 +1423,67 @@ class Qdrant(VectorStore):
                 timeout=timeout,  # type: ignore[arg-type]
             )
             is_new_collection = True
+        if force_recreate:
+            raise ValueError
+
+        # Get the vector configuration of the existing collection and vector, if it
+        # was specified. If the old configuration does not match the current one,
+        # an exception is being thrown.
+        collection_info = client.get_collection(collection_name=collection_name)
+        current_vector_config = collection_info.config.params.vectors
+        if isinstance(current_vector_config, dict) and vector_name is not None:
+            if vector_name not in current_vector_config:
+                raise QdrantException(
+                    f"Existing Qdrant collection {collection_name} does not "
+                    f"contain vector named {vector_name}. Did you mean one of the "
+                    f"existing vectors: {', '.join(current_vector_config.keys())}? "
+                    f"If you want to recreate the collection, set `force_recreate` "
+                    f"parameter to `True`."
+                )
+            current_vector_config = current_vector_config.get(
+                vector_name
+            )  # type: ignore[assignment]
+        elif isinstance(current_vector_config, dict) and vector_name is None:
+            raise QdrantException(
+                f"Existing Qdrant collection {collection_name} uses named vectors. "
+                f"If you want to reuse it, please set `vector_name` to any of the "
+                f"existing named vectors: "
+                f"{', '.join(current_vector_config.keys())}."  # noqa
+                f"If you want to recreate the collection, set `force_recreate` "
+                f"parameter to `True`."
+            )
+        elif (
+                not isinstance(current_vector_config, dict) and vector_name is not None
+        ):
+            raise QdrantException(
+                f"Existing Qdrant collection {collection_name} doesn't use named "
+                f"vectors. If you want to reuse it, please set `vector_name` to "
+                f"`None`. If you want to recreate the collection, set "
+                f"`force_recreate` parameter to `True`."
+            )
+
+        # Check if the vector configuration has the same dimensionality.
+        if current_vector_config.size != vector_size:  # type: ignore[union-attr]
+            raise QdrantException(
+                f"Existing Qdrant collection is configured for vectors with "
+                f"{current_vector_config.size} "  # type: ignore[union-attr]
+                f"dimensions. Selected embeddings are {vector_size}-dimensional. "
+                f"If you want to recreate the collection, set `force_recreate` "
+                f"parameter to `True`."
+            )
+
+        current_distance_func = (
+            current_vector_config.distance.name.upper()  # type: ignore[union-attr]
+        )
+        if current_distance_func != distance_func:
+            raise QdrantException(
+                f"Existing Qdrant collection is configured for "
+                f"{current_vector_config.distance} "  # type: ignore[union-attr]
+                f"similarity. Please set `distance_func` parameter to "
+                f"`{distance_func}` if you want to reuse it. If you want to "
+                f"recreate the collection, set `force_recreate` parameter to "
+                f"`True`."
+            )
         qdrant = cls(
             client=client,
             collection_name=collection_name,

api/core/index/vector_index/qdrant_vector_index.py

@@ -169,6 +169,19 @@ class QdrantVectorIndex(BaseVectorIndex):
             ],
         ))
 
+    def delete(self) -> None:
+        vector_store = self._get_vector_store()
+        vector_store = cast(self._get_vector_store_class(), vector_store)
+
+        from qdrant_client.http import models
+        vector_store.del_texts(models.Filter(
+            must=[
+                models.FieldCondition(
+                    key="group_id",
+                    match=models.MatchValue(value=self.dataset.id),
+                ),
+            ],
+        ))
 
     def _is_origin(self):
         if self.dataset.index_struct_dict:

api/core/login/login.py

@@ -1,11 +1,10 @@
 import os
 from functools import wraps
 
-import flask_login
 from flask import current_app
 from flask import g
 from flask import has_request_context
-from flask import request
+from flask import request, session
 from flask_login import user_logged_in
 from flask_login.config import EXEMPT_METHODS
 from werkzeug.exceptions import Unauthorized

api/core/model_providers/models/embedding/huggingface_embedding.py

@@ -0,0 +1,22 @@
+from core.model_providers.error import LLMBadRequestError
+from core.model_providers.providers.base import BaseModelProvider
+from core.third_party.langchain.embeddings.huggingface_hub_embedding import HuggingfaceHubEmbeddings
+from core.model_providers.models.embedding.base import BaseEmbedding
+
+
+class HuggingfaceEmbedding(BaseEmbedding):
+    def __init__(self, model_provider: BaseModelProvider, name: str):
+        credentials = model_provider.get_model_credentials(
+            model_name=name,
+            model_type=self.type
+        )
+
+        client = HuggingfaceHubEmbeddings(
+            model=name,
+            **credentials
+        )
+
+        super().__init__(model_provider, client, name)
+
+    def handle_exceptions(self, ex: Exception) -> Exception:
+        return LLMBadRequestError(f"Huggingface embedding: {str(ex)}")

api/core/model_providers/models/llm/base.py

@@ -1,6 +1,7 @@
 import json
 import os
 import re
+import time
 from abc import abstractmethod
 from typing import List, Optional, Any, Union, Tuple
 import decimal
@@ -20,6 +21,8 @@ from core.prompt.prompt_template import JinjaPromptTemplate
 from core.third_party.langchain.llms.fake import FakeLLM
 import logging
 
+from extensions.ext_database import db
+
 logger = logging.getLogger(__name__)
 
 
@@ -129,6 +132,8 @@ class BaseLLM(BaseProviderModel):
         if self.deduct_quota:
             self.model_provider.check_quota_over_limit()
 
+        db.session.commit()
+
         if not callbacks:
             callbacks = self.callbacks
         else:

api/core/model_providers/models/llm/wenxin_model.py

@@ -18,6 +18,7 @@ class WenxinModel(BaseLLM):
         provider_model_kwargs = self._to_model_kwargs_input(self.model_rules, self.model_kwargs)
         # TODO load price_config from configs(db)
         return Wenxin(
+            model=self.name,
             streaming=self.streaming,
             callbacks=self.callbacks,
             **self.credentials,

api/core/model_providers/providers/huggingface_hub_provider.py

@@ -1,5 +1,6 @@
 import json
 from typing import Type
+import requests
 
 from huggingface_hub import HfApi
 
@@ -10,8 +11,12 @@ from core.model_providers.providers.base import BaseModelProvider, CredentialsVa
 
 from core.model_providers.models.base import BaseProviderModel
 from core.third_party.langchain.llms.huggingface_endpoint_llm import HuggingFaceEndpointLLM
+from core.third_party.langchain.embeddings.huggingface_hub_embedding import HuggingfaceHubEmbeddings
+from core.model_providers.models.embedding.huggingface_embedding import HuggingfaceEmbedding
 from models.provider import ProviderType
 
+HUGGINGFACE_ENDPOINT_API = 'https://api.endpoints.huggingface.cloud/v2/endpoint/'
+
 
 class HuggingfaceHubProvider(BaseModelProvider):
     @property
@@ -33,6 +38,8 @@ class HuggingfaceHubProvider(BaseModelProvider):
         """
         if model_type == ModelType.TEXT_GENERATION:
             model_class = HuggingfaceHubModel
+        elif model_type == ModelType.EMBEDDINGS:
+            model_class = HuggingfaceEmbedding
         else:
             raise NotImplementedError
 
@@ -63,7 +70,7 @@ class HuggingfaceHubProvider(BaseModelProvider):
         :param model_type:
         :param credentials:
         """
-        if model_type != ModelType.TEXT_GENERATION:
+        if model_type not in [ModelType.TEXT_GENERATION, ModelType.EMBEDDINGS]:
             raise NotImplementedError
 
         if 'huggingfacehub_api_type' not in credentials \
@@ -88,19 +95,15 @@ class HuggingfaceHubProvider(BaseModelProvider):
             if 'task_type' not in credentials:
                 raise CredentialsValidateFailedError('Task Type must be provided.')
 
-            if credentials['task_type'] not in ("text2text-generation", "text-generation", "summarization"):
+            if credentials['task_type'] not in ("text2text-generation", "text-generation", 'feature-extraction'):
                 raise CredentialsValidateFailedError('Task Type must be one of text2text-generation, '
-                                                     'text-generation, summarization.')
+                                                     'text-generation, feature-extraction.')
 
             try:
-                llm = HuggingFaceEndpointLLM(
-                    endpoint_url=credentials['huggingfacehub_endpoint_url'],
-                    task=credentials['task_type'],
-                    model_kwargs={"temperature": 0.5, "max_new_tokens": 200},
-                    huggingfacehub_api_token=credentials['huggingfacehub_api_token']
-                )
-
-                llm("ping")
+                if credentials['task_type'] == 'feature-extraction':
+                    cls.check_embedding_valid(credentials, model_name)
+                else:
+                    cls.check_llm_valid(credentials)    
             except Exception as e:
                 raise CredentialsValidateFailedError(f"{e.__class__.__name__}:{str(e)}")
         else:
@@ -112,13 +115,64 @@ class HuggingfaceHubProvider(BaseModelProvider):
                 if 'inference' in model_info.cardData and not model_info.cardData['inference']:
                     raise ValueError(f'Inference API has been turned off for this model {model_name}.')
 
-                VALID_TASKS = ("text2text-generation", "text-generation", "summarization")
+                VALID_TASKS = ("text2text-generation", "text-generation", "feature-extraction")
                 if model_info.pipeline_tag not in VALID_TASKS:
                     raise ValueError(f"Model {model_name} is not a valid task, "
                                      f"must be one of {VALID_TASKS}.")
             except Exception as e:
                 raise CredentialsValidateFailedError(f"{e.__class__.__name__}:{str(e)}")
 
+    @classmethod
+    def check_llm_valid(cls, credentials: dict):
+        llm = HuggingFaceEndpointLLM(
+            endpoint_url=credentials['huggingfacehub_endpoint_url'],
+            task=credentials['task_type'],
+            model_kwargs={"temperature": 0.5, "max_new_tokens": 200},
+            huggingfacehub_api_token=credentials['huggingfacehub_api_token']
+        )
+
+        llm("ping")
+
+    @classmethod
+    def check_embedding_valid(cls, credentials: dict, model_name: str):
+
+        cls.check_endpoint_url_model_repository_name(credentials, model_name)
+        
+        embedding_model = HuggingfaceHubEmbeddings(
+            model=model_name,
+            **credentials
+        )
+
+        embedding_model.embed_query("ping")
+
+    @classmethod
+    def check_endpoint_url_model_repository_name(cls, credentials: dict, model_name: str):
+        try:
+            url = f'{HUGGINGFACE_ENDPOINT_API}{credentials["huggingface_namespace"]}'
+            headers = {
+                'Authorization': f'Bearer {credentials["huggingfacehub_api_token"]}',
+                'Content-Type': 'application/json'
+            }
+
+            response =requests.get(url=url, headers=headers)
+
+            if response.status_code != 200:
+                raise ValueError('User Name or Organization Name is invalid.')
+
+            model_repository_name = ''
+
+            for item in response.json().get("items", []):
+                if item.get("status", {}).get("url") == credentials['huggingfacehub_endpoint_url']:
+                    model_repository_name = item.get("model", {}).get("repository")
+                    break
+            
+            if model_repository_name != model_name:
+                raise ValueError(f'Model Name {model_name} is invalid. Please check it on the inference endpoints console.')
+
+        except Exception as e:
+            raise ValueError(str(e))
+        
+
     @classmethod
     def encrypt_model_credentials(cls, tenant_id: str, model_name: str, model_type: ModelType,
                                   credentials: dict) -> dict:

api/core/model_providers/providers/wenxin_provider.py

@@ -61,13 +61,18 @@ class WenxinProvider(BaseModelProvider):
         :param model_type:
         :return:
         """
+        model_max_tokens = {
+            'ernie-bot': 4800,
+            'ernie-bot-turbo': 11200,
+        }
+
         if model_name in ['ernie-bot', 'ernie-bot-turbo']:
             return ModelKwargsRules(
                 temperature=KwargRule[float](min=0.01, max=1, default=0.95, precision=2),
                 top_p=KwargRule[float](min=0.01, max=1, default=0.8, precision=2),
                 presence_penalty=KwargRule[float](enabled=False),
                 frequency_penalty=KwargRule[float](enabled=False),
-                max_tokens=KwargRule[int](enabled=False),
+                max_tokens=KwargRule[int](enabled=False, max=model_max_tokens.get(model_name)),
             )
         else:
             return ModelKwargsRules(

api/core/third_party/langchain/embeddings/huggingface_hub_embedding.py

@@ -0,0 +1,74 @@
+from typing import Any, Dict, List, Optional
+import json
+import numpy as np
+
+from pydantic import BaseModel, Extra, root_validator
+
+from langchain.embeddings.base import Embeddings
+from langchain.utils import get_from_dict_or_env
+from huggingface_hub import InferenceClient
+
+HOSTED_INFERENCE_API = 'hosted_inference_api'
+INFERENCE_ENDPOINTS = 'inference_endpoints'
+
+
+class HuggingfaceHubEmbeddings(BaseModel, Embeddings):
+    client: Any
+    model: str
+
+    huggingface_namespace: Optional[str] = None
+    task_type: Optional[str] = None
+    huggingfacehub_api_type: Optional[str] = None
+    huggingfacehub_api_token: Optional[str] = None
+    huggingfacehub_endpoint_url: Optional[str] = None
+
+    class Config:
+        extra = Extra.forbid
+
+    @root_validator()
+    def validate_environment(cls, values: Dict) -> Dict:
+        values['huggingfacehub_api_token'] = get_from_dict_or_env(
+            values, "huggingfacehub_api_token", "HUGGINGFACEHUB_API_TOKEN"
+        )
+
+        values['client'] = InferenceClient(token=values['huggingfacehub_api_token'])
+
+        return values
+
+    def embed_documents(self, texts: List[str]) -> List[List[float]]:
+        model = ''
+
+        if self.huggingfacehub_api_type == HOSTED_INFERENCE_API:
+            model = self.model
+        else:
+            model = self.huggingfacehub_endpoint_url
+
+        output = self.client.post(
+            json={
+                "inputs": texts,
+                "options": {
+                    "wait_for_model": False,
+                    "use_cache": False
+                }
+            }, model=model)
+        
+        embeddings =  json.loads(output.decode())
+        return self.mean_pooling(embeddings)
+
+    def embed_query(self, text: str) -> List[float]:
+        return self.embed_documents([text])[0]
+    
+    # https://huggingface.co/docs/api-inference/detailed_parameters#feature-extraction-task
+    # Returned values are a list of floats, or a list of list of floats 
+    # (depending on if you sent a string or a list of string, 
+    # and if the automatic reduction, usually mean_pooling for instance was applied for you or not. 
+    # This should be explained on the model's README.)
+    def mean_pooling(self, embeddings: List) -> List[float]:
+        # If automatic reduction by giving model, no need to mean_pooling.
+        # For example one: List[List[float]]
+        if not isinstance(embeddings[0][0], list):
+            return embeddings
+
+        # For example two: List[List[List[float]]], need to mean_pooling.
+        sentence_embeddings = [np.mean(embedding[0], axis=0).tolist() for embedding in embeddings]
+        return sentence_embeddings

api/core/third_party/langchain/llms/huggingface_hub_llm.py

@@ -16,7 +16,7 @@ class HuggingFaceHubLLM(HuggingFaceHub):
     environment variable ``HUGGINGFACEHUB_API_TOKEN`` set with your API token, or pass
     it as a named parameter to the constructor.
 
-    Only supports `text-generation`, `text2text-generation` and `summarization` for now.
+    Only supports `text-generation`, `text2text-generation` for now.
 
     Example:
         .. code-block:: python

api/events/event_handlers/clean_when_dataset_deleted.py

@@ -5,4 +5,5 @@ from tasks.clean_dataset_task import clean_dataset_task
 @dataset_was_deleted.connect
 def handle(sender, **kwargs):
     dataset = sender
-    clean_dataset_task.delay(dataset.id, dataset.tenant_id, dataset.indexing_technique, dataset.index_struct)
+    clean_dataset_task.delay(dataset.id, dataset.tenant_id, dataset.indexing_technique,
+                             dataset.index_struct, dataset.collection_binding_id)

api/extensions/ext_session.py

@@ -1,174 +0,0 @@
-import redis
-from redis.connection import SSLConnection, Connection
-from flask import request
-from flask_session import Session, SqlAlchemySessionInterface, RedisSessionInterface
-from flask_session.sessions import total_seconds
-from itsdangerous import want_bytes
-
-from extensions.ext_database import db
-
-sess = Session()
-
-
-def init_app(app):
-    sqlalchemy_session_interface = CustomSqlAlchemySessionInterface(
-        app,
-        db,
-        app.config.get('SESSION_SQLALCHEMY_TABLE', 'sessions'),
-        app.config.get('SESSION_KEY_PREFIX', 'session:'),
-        app.config.get('SESSION_USE_SIGNER', False),
-        app.config.get('SESSION_PERMANENT', True)
-    )
-
-    session_type = app.config.get('SESSION_TYPE')
-    if session_type == 'sqlalchemy':
-        app.session_interface = sqlalchemy_session_interface
-    elif session_type == 'redis':
-        connection_class = Connection
-        if app.config.get('SESSION_REDIS_USE_SSL', False):
-            connection_class = SSLConnection
-
-        sess_redis_client = redis.Redis()
-        sess_redis_client.connection_pool = redis.ConnectionPool(**{
-            'host': app.config.get('SESSION_REDIS_HOST', 'localhost'),
-            'port': app.config.get('SESSION_REDIS_PORT', 6379),
-            'username': app.config.get('SESSION_REDIS_USERNAME', None),
-            'password': app.config.get('SESSION_REDIS_PASSWORD', None),
-            'db': app.config.get('SESSION_REDIS_DB', 2),
-            'encoding': 'utf-8',
-            'encoding_errors': 'strict',
-            'decode_responses': False
-        }, connection_class=connection_class)
-
-        app.extensions['session_redis'] = sess_redis_client
-
-        app.session_interface = CustomRedisSessionInterface(
-            sess_redis_client,
-            app.config.get('SESSION_KEY_PREFIX', 'session:'),
-            app.config.get('SESSION_USE_SIGNER', False),
-            app.config.get('SESSION_PERMANENT', True)
-        )
-
-
-class CustomSqlAlchemySessionInterface(SqlAlchemySessionInterface):
-
-    def __init__(
-        self,
-        app,
-        db,
-        table,
-        key_prefix,
-        use_signer=False,
-        permanent=True,
-        sequence=None,
-        autodelete=False,
-    ):
-        if db is None:
-            from flask_sqlalchemy import SQLAlchemy
-
-            db = SQLAlchemy(app)
-        self.db = db
-        self.key_prefix = key_prefix
-        self.use_signer = use_signer
-        self.permanent = permanent
-        self.autodelete = autodelete
-        self.sequence = sequence
-        self.has_same_site_capability = hasattr(self, "get_cookie_samesite")
-
-        class Session(self.db.Model):
-            __tablename__ = table
-
-            if sequence:
-                id = self.db.Column(  # noqa: A003, VNE003, A001
-                    self.db.Integer, self.db.Sequence(sequence), primary_key=True
-                )
-            else:
-                id = self.db.Column(  # noqa: A003, VNE003, A001
-                    self.db.Integer, primary_key=True
-                )
-
-            session_id = self.db.Column(self.db.String(255), unique=True)
-            data = self.db.Column(self.db.LargeBinary)
-            expiry = self.db.Column(self.db.DateTime)
-
-            def __init__(self, session_id, data, expiry):
-                self.session_id = session_id
-                self.data = data
-                self.expiry = expiry
-
-            def __repr__(self):
-                return f"<Session data {self.data}>"
-
-        self.sql_session_model = Session
-
-    def save_session(self, *args, **kwargs):
-        if request.blueprint == 'service_api':
-            return
-        elif request.method == 'OPTIONS':
-            return
-        elif request.endpoint and request.endpoint == 'health':
-            return
-        return super().save_session(*args, **kwargs)
-
-
-class CustomRedisSessionInterface(RedisSessionInterface):
-
-    def save_session(self, app, session, response):
-        if request.blueprint == 'service_api':
-            return
-        elif request.method == 'OPTIONS':
-            return
-        elif request.endpoint and request.endpoint == 'health':
-            return
-
-        if not self.should_set_cookie(app, session):
-            return
-        domain = self.get_cookie_domain(app)
-        path = self.get_cookie_path(app)
-        if not session:
-            if session.modified:
-                self.redis.delete(self.key_prefix + session.sid)
-                response.delete_cookie(
-                    app.config["SESSION_COOKIE_NAME"], domain=domain, path=path
-                )
-            return
-
-        # Modification case.  There are upsides and downsides to
-        # emitting a set-cookie header each request.  The behavior
-        # is controlled by the :meth:`should_set_cookie` method
-        # which performs a quick check to figure out if the cookie
-        # should be set or not.  This is controlled by the
-        # SESSION_REFRESH_EACH_REQUEST config flag as well as
-        # the permanent flag on the session itself.
-        # if not self.should_set_cookie(app, session):
-        #    return
-        conditional_cookie_kwargs = {}
-        httponly = self.get_cookie_httponly(app)
-        secure = self.get_cookie_secure(app)
-        if self.has_same_site_capability:
-            conditional_cookie_kwargs["samesite"] = self.get_cookie_samesite(app)
-        expires = self.get_expiration_time(app, session)
-
-        if session.permanent:
-            value = self.serializer.dumps(dict(session))
-            if value is not None:
-                self.redis.setex(
-                    name=self.key_prefix + session.sid,
-                    value=value,
-                    time=total_seconds(app.permanent_session_lifetime),
-                )
-
-        if self.use_signer:
-            session_id = self._get_signer(app).sign(want_bytes(session.sid)).decode("utf-8")
-        else:
-            session_id = session.sid
-        response.set_cookie(
-            app.config["SESSION_COOKIE_NAME"],
-            session_id,
-            expires=expires,
-            httponly=httponly,
-            domain=domain,
-            path=path,
-            secure=secure,
-            **conditional_cookie_kwargs,
-        )

api/fields/app_fields.py

@@ -0,0 +1,138 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+app_detail_kernel_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+}
+
+related_app_list = {
+    'data': fields.List(fields.Nested(app_detail_kernel_fields)),
+    'total': fields.Integer,
+}
+
+model_config_fields = {
+    'opening_statement': fields.String,
+    'suggested_questions': fields.Raw(attribute='suggested_questions_list'),
+    'suggested_questions_after_answer': fields.Raw(attribute='suggested_questions_after_answer_dict'),
+    'speech_to_text': fields.Raw(attribute='speech_to_text_dict'),
+    'retriever_resource': fields.Raw(attribute='retriever_resource_dict'),
+    'more_like_this': fields.Raw(attribute='more_like_this_dict'),
+    'sensitive_word_avoidance': fields.Raw(attribute='sensitive_word_avoidance_dict'),
+    'model': fields.Raw(attribute='model_dict'),
+    'user_input_form': fields.Raw(attribute='user_input_form_list'),
+    'dataset_query_variable': fields.String,
+    'pre_prompt': fields.String,
+    'agent_mode': fields.Raw(attribute='agent_mode_dict'),
+}
+
+app_detail_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'enable_site': fields.Boolean,
+    'enable_api': fields.Boolean,
+    'api_rpm': fields.Integer,
+    'api_rph': fields.Integer,
+    'is_demo': fields.Boolean,
+    'model_config': fields.Nested(model_config_fields, attribute='app_model_config'),
+    'created_at': TimestampField
+}
+
+prompt_config_fields = {
+    'prompt_template': fields.String,
+}
+
+model_config_partial_fields = {
+    'model': fields.Raw(attribute='model_dict'),
+    'pre_prompt': fields.String,
+}
+
+app_partial_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'enable_site': fields.Boolean,
+    'enable_api': fields.Boolean,
+    'is_demo': fields.Boolean,
+    'model_config': fields.Nested(model_config_partial_fields, attribute='app_model_config'),
+    'created_at': TimestampField
+}
+
+app_pagination_fields = {
+    'page': fields.Integer,
+    'limit': fields.Integer(attribute='per_page'),
+    'total': fields.Integer,
+    'has_more': fields.Boolean(attribute='has_next'),
+    'data': fields.List(fields.Nested(app_partial_fields), attribute='items')
+}
+
+template_fields = {
+    'name': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'description': fields.String,
+    'mode': fields.String,
+    'model_config': fields.Nested(model_config_fields),
+}
+
+template_list_fields = {
+    'data': fields.List(fields.Nested(template_fields)),
+}
+
+site_fields = {
+    'access_token': fields.String(attribute='code'),
+    'code': fields.String,
+    'title': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'description': fields.String,
+    'default_language': fields.String,
+    'customize_domain': fields.String,
+    'copyright': fields.String,
+    'privacy_policy': fields.String,
+    'customize_token_strategy': fields.String,
+    'prompt_public': fields.Boolean,
+    'app_base_url': fields.String,
+}
+
+app_detail_fields_with_site = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'enable_site': fields.Boolean,
+    'enable_api': fields.Boolean,
+    'api_rpm': fields.Integer,
+    'api_rph': fields.Integer,
+    'is_demo': fields.Boolean,
+    'model_config': fields.Nested(model_config_fields, attribute='app_model_config'),
+    'site': fields.Nested(site_fields),
+    'api_base_url': fields.String,
+    'created_at': TimestampField
+}
+
+app_site_fields = {
+    'app_id': fields.String,
+    'access_token': fields.String(attribute='code'),
+    'code': fields.String,
+    'title': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String,
+    'description': fields.String,
+    'default_language': fields.String,
+    'customize_domain': fields.String,
+    'copyright': fields.String,
+    'privacy_policy': fields.String,
+    'customize_token_strategy': fields.String,
+    'prompt_public': fields.Boolean
+}

api/fields/conversation_fields.py

@@ -0,0 +1,182 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+
+class MessageTextField(fields.Raw):
+    def format(self, value):
+        return value[0]['text'] if value else ''
+
+
+account_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'email': fields.String
+}
+
+feedback_fields = {
+    'rating': fields.String,
+    'content': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account': fields.Nested(account_fields, allow_null=True),
+}
+
+annotation_fields = {
+    'content': fields.String,
+    'account': fields.Nested(account_fields, allow_null=True),
+    'created_at': TimestampField
+}
+
+message_detail_fields = {
+    'id': fields.String,
+    'conversation_id': fields.String,
+    'inputs': fields.Raw,
+    'query': fields.String,
+    'message': fields.Raw,
+    'message_tokens': fields.Integer,
+    'answer': fields.String,
+    'answer_tokens': fields.Integer,
+    'provider_response_latency': fields.Float,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account_id': fields.String,
+    'feedbacks': fields.List(fields.Nested(feedback_fields)),
+    'annotation': fields.Nested(annotation_fields, allow_null=True),
+    'created_at': TimestampField
+}
+
+feedback_stat_fields = {
+    'like': fields.Integer,
+    'dislike': fields.Integer
+}
+
+model_config_fields = {
+    'opening_statement': fields.String,
+    'suggested_questions': fields.Raw,
+    'model': fields.Raw,
+    'user_input_form': fields.Raw,
+    'pre_prompt': fields.String,
+    'agent_mode': fields.Raw,
+}
+
+simple_configs_fields = {
+    'prompt_template': fields.String,
+}
+
+simple_model_config_fields = {
+    'model': fields.Raw(attribute='model_dict'),
+    'pre_prompt': fields.String,
+}
+
+simple_message_detail_fields = {
+    'inputs': fields.Raw,
+    'query': fields.String,
+    'message': MessageTextField,
+    'answer': fields.String,
+}
+
+conversation_fields = {
+    'id': fields.String,
+    'status': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_end_user_session_id': fields.String(),
+    'from_account_id': fields.String,
+    'read_at': TimestampField,
+    'created_at': TimestampField,
+    'annotation': fields.Nested(annotation_fields, allow_null=True),
+    'model_config': fields.Nested(simple_model_config_fields),
+    'user_feedback_stats': fields.Nested(feedback_stat_fields),
+    'admin_feedback_stats': fields.Nested(feedback_stat_fields),
+    'message': fields.Nested(simple_message_detail_fields, attribute='first_message')
+}
+
+conversation_pagination_fields = {
+    'page': fields.Integer,
+    'limit': fields.Integer(attribute='per_page'),
+    'total': fields.Integer,
+    'has_more': fields.Boolean(attribute='has_next'),
+    'data': fields.List(fields.Nested(conversation_fields), attribute='items')
+}
+
+conversation_message_detail_fields = {
+    'id': fields.String,
+    'status': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account_id': fields.String,
+    'created_at': TimestampField,
+    'model_config': fields.Nested(model_config_fields),
+    'message': fields.Nested(message_detail_fields, attribute='first_message'),
+}
+
+simple_model_config_fields = {
+    'model': fields.Raw(attribute='model_dict'),
+    'pre_prompt': fields.String,
+}
+
+conversation_with_summary_fields = {
+    'id': fields.String,
+    'status': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_end_user_session_id': fields.String,
+    'from_account_id': fields.String,
+    'summary': fields.String(attribute='summary_or_query'),
+    'read_at': TimestampField,
+    'created_at': TimestampField,
+    'annotated': fields.Boolean,
+    'model_config': fields.Nested(simple_model_config_fields),
+    'message_count': fields.Integer,
+    'user_feedback_stats': fields.Nested(feedback_stat_fields),
+    'admin_feedback_stats': fields.Nested(feedback_stat_fields)
+}
+
+conversation_with_summary_pagination_fields = {
+    'page': fields.Integer,
+    'limit': fields.Integer(attribute='per_page'),
+    'total': fields.Integer,
+    'has_more': fields.Boolean(attribute='has_next'),
+    'data': fields.List(fields.Nested(conversation_with_summary_fields), attribute='items')
+}
+
+conversation_detail_fields = {
+    'id': fields.String,
+    'status': fields.String,
+    'from_source': fields.String,
+    'from_end_user_id': fields.String,
+    'from_account_id': fields.String,
+    'created_at': TimestampField,
+    'annotated': fields.Boolean,
+    'model_config': fields.Nested(model_config_fields),
+    'message_count': fields.Integer,
+    'user_feedback_stats': fields.Nested(feedback_stat_fields),
+    'admin_feedback_stats': fields.Nested(feedback_stat_fields)
+}
+
+simple_conversation_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'inputs': fields.Raw,
+    'status': fields.String,
+    'introduction': fields.String,
+    'created_at': TimestampField
+}
+
+conversation_infinite_scroll_pagination_fields = {
+    'limit': fields.Integer,
+    'has_more': fields.Boolean,
+    'data': fields.List(fields.Nested(simple_conversation_fields))
+}
+
+conversation_with_model_config_fields = {
+    **simple_conversation_fields,
+    'model_config': fields.Raw,
+}
+
+conversation_with_model_config_infinite_scroll_pagination_fields = {
+    'limit': fields.Integer,
+    'has_more': fields.Boolean,
+    'data': fields.List(fields.Nested(conversation_with_model_config_fields))
+}

api/fields/data_source_fields.py

@@ -0,0 +1,65 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+integrate_icon_fields = {
+    'type': fields.String,
+    'url': fields.String,
+    'emoji': fields.String
+}
+
+integrate_page_fields = {
+    'page_name': fields.String,
+    'page_id': fields.String,
+    'page_icon': fields.Nested(integrate_icon_fields, allow_null=True),
+    'is_bound': fields.Boolean,
+    'parent_id': fields.String,
+    'type': fields.String
+}
+
+integrate_workspace_fields = {
+    'workspace_name': fields.String,
+    'workspace_id': fields.String,
+    'workspace_icon': fields.String,
+    'pages': fields.List(fields.Nested(integrate_page_fields))
+}
+
+integrate_notion_info_list_fields = {
+    'notion_info': fields.List(fields.Nested(integrate_workspace_fields)),
+}
+
+integrate_icon_fields = {
+    'type': fields.String,
+    'url': fields.String,
+    'emoji': fields.String
+}
+
+integrate_page_fields = {
+    'page_name': fields.String,
+    'page_id': fields.String,
+    'page_icon': fields.Nested(integrate_icon_fields, allow_null=True),
+    'parent_id': fields.String,
+    'type': fields.String
+}
+
+integrate_workspace_fields = {
+    'workspace_name': fields.String,
+    'workspace_id': fields.String,
+    'workspace_icon': fields.String,
+    'pages': fields.List(fields.Nested(integrate_page_fields)),
+    'total': fields.Integer
+}
+
+integrate_fields = {
+    'id': fields.String,
+    'provider': fields.String,
+    'created_at': TimestampField,
+    'is_bound': fields.Boolean,
+    'disabled': fields.Boolean,
+    'link': fields.String,
+    'source_info': fields.Nested(integrate_workspace_fields)
+}
+
+integrate_list_fields = {
+    'data': fields.List(fields.Nested(integrate_fields)),
+}

api/fields/dataset_fields.py

@@ -0,0 +1,43 @@
+from flask_restful import fields
+from libs.helper import TimestampField
+
+dataset_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'description': fields.String,
+    'permission': fields.String,
+    'data_source_type': fields.String,
+    'indexing_technique': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+}
+
+dataset_detail_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'description': fields.String,
+    'provider': fields.String,
+    'permission': fields.String,
+    'data_source_type': fields.String,
+    'indexing_technique': fields.String,
+    'app_count': fields.Integer,
+    'document_count': fields.Integer,
+    'word_count': fields.Integer,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'updated_by': fields.String,
+    'updated_at': TimestampField,
+    'embedding_model': fields.String,
+    'embedding_model_provider': fields.String,
+    'embedding_available': fields.Boolean
+}
+
+dataset_query_detail_fields = {
+    "id": fields.String,
+    "content": fields.String,
+    "source": fields.String,
+    "source_app_id": fields.String,
+    "created_by_role": fields.String,
+    "created_by": fields.String,
+    "created_at": TimestampField
+}

api/fields/document_fields.py

@@ -0,0 +1,76 @@
+from flask_restful import fields
+
+from fields.dataset_fields import dataset_fields
+from libs.helper import TimestampField
+
+document_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'data_source_type': fields.String,
+    'data_source_info': fields.Raw(attribute='data_source_info_dict'),
+    'dataset_process_rule_id': fields.String,
+    'name': fields.String,
+    'created_from': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'tokens': fields.Integer,
+    'indexing_status': fields.String,
+    'error': fields.String,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'archived': fields.Boolean,
+    'display_status': fields.String,
+    'word_count': fields.Integer,
+    'hit_count': fields.Integer,
+    'doc_form': fields.String,
+}
+
+document_with_segments_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'data_source_type': fields.String,
+    'data_source_info': fields.Raw(attribute='data_source_info_dict'),
+    'dataset_process_rule_id': fields.String,
+    'name': fields.String,
+    'created_from': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'tokens': fields.Integer,
+    'indexing_status': fields.String,
+    'error': fields.String,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'archived': fields.Boolean,
+    'display_status': fields.String,
+    'word_count': fields.Integer,
+    'hit_count': fields.Integer,
+    'completed_segments': fields.Integer,
+    'total_segments': fields.Integer
+}
+
+dataset_and_document_fields = {
+    'dataset': fields.Nested(dataset_fields),
+    'documents': fields.List(fields.Nested(document_fields)),
+    'batch': fields.String
+}
+
+document_status_fields = {
+    'id': fields.String,
+    'indexing_status': fields.String,
+    'processing_started_at': TimestampField,
+    'parsing_completed_at': TimestampField,
+    'cleaning_completed_at': TimestampField,
+    'splitting_completed_at': TimestampField,
+    'completed_at': TimestampField,
+    'paused_at': TimestampField,
+    'error': fields.String,
+    'stopped_at': TimestampField,
+    'completed_segments': fields.Integer,
+    'total_segments': fields.Integer,
+}
+
+document_status_fields_list = {
+    'data': fields.List(fields.Nested(document_status_fields))
+}

api/fields/file_fields.py

@@ -0,0 +1,18 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+upload_config_fields = {
+    'file_size_limit': fields.Integer,
+    'batch_count_limit': fields.Integer
+}
+
+file_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'size': fields.Integer,
+    'extension': fields.String,
+    'mime_type': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+}

api/fields/hit_testing_fields.py

@@ -0,0 +1,41 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+document_fields = {
+    'id': fields.String,
+    'data_source_type': fields.String,
+    'name': fields.String,
+    'doc_type': fields.String,
+}
+
+segment_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'document_id': fields.String,
+    'content': fields.String,
+    'answer': fields.String,
+    'word_count': fields.Integer,
+    'tokens': fields.Integer,
+    'keywords': fields.List(fields.String),
+    'index_node_id': fields.String,
+    'index_node_hash': fields.String,
+    'hit_count': fields.Integer,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'status': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'indexing_at': TimestampField,
+    'completed_at': TimestampField,
+    'error': fields.String,
+    'stopped_at': TimestampField,
+    'document': fields.Nested(document_fields),
+}
+
+hit_testing_record_fields = {
+    'segment': fields.Nested(segment_fields),
+    'score': fields.Float,
+    'tsne_position': fields.Raw
+}

api/fields/installed_app_fields.py

@@ -0,0 +1,25 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+app_fields = {
+    'id': fields.String,
+    'name': fields.String,
+    'mode': fields.String,
+    'icon': fields.String,
+    'icon_background': fields.String
+}
+
+installed_app_fields = {
+    'id': fields.String,
+    'app': fields.Nested(app_fields),
+    'app_owner_tenant_id': fields.String,
+    'is_pinned': fields.Boolean,
+    'last_used_at': TimestampField,
+    'editable': fields.Boolean,
+    'uninstallable': fields.Boolean,
+}
+
+installed_app_list_fields = {
+    'installed_apps': fields.List(fields.Nested(installed_app_fields))
+}

api/fields/message_fields.py

@@ -0,0 +1,43 @@
+from flask_restful import fields
+
+from libs.helper import TimestampField
+
+feedback_fields = {
+    'rating': fields.String
+}
+
+retriever_resource_fields = {
+    'id': fields.String,
+    'message_id': fields.String,
+    'position': fields.Integer,
+    'dataset_id': fields.String,
+    'dataset_name': fields.String,
+    'document_id': fields.String,
+    'document_name': fields.String,
+    'data_source_type': fields.String,
+    'segment_id': fields.String,
+    'score': fields.Float,
+    'hit_count': fields.Integer,
+    'word_count': fields.Integer,
+    'segment_position': fields.Integer,
+    'index_node_hash': fields.String,
+    'content': fields.String,
+    'created_at': TimestampField
+}
+
+message_fields = {
+    'id': fields.String,
+    'conversation_id': fields.String,
+    'inputs': fields.Raw,
+    'query': fields.String,
+    'answer': fields.String,
+    'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
+    'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
+    'created_at': TimestampField
+}
+
+message_infinite_scroll_pagination_fields = {
+    'limit': fields.Integer,
+    'has_more': fields.Boolean,
+    'data': fields.List(fields.Nested(message_fields))
+}

api/fields/segment_fields.py

@@ -0,0 +1,32 @@
+from flask_restful import fields
+from libs.helper import TimestampField
+
+segment_fields = {
+    'id': fields.String,
+    'position': fields.Integer,
+    'document_id': fields.String,
+    'content': fields.String,
+    'answer': fields.String,
+    'word_count': fields.Integer,
+    'tokens': fields.Integer,
+    'keywords': fields.List(fields.String),
+    'index_node_id': fields.String,
+    'index_node_hash': fields.String,
+    'hit_count': fields.Integer,
+    'enabled': fields.Boolean,
+    'disabled_at': TimestampField,
+    'disabled_by': fields.String,
+    'status': fields.String,
+    'created_by': fields.String,
+    'created_at': TimestampField,
+    'indexing_at': TimestampField,
+    'completed_at': TimestampField,
+    'error': fields.String,
+    'stopped_at': TimestampField
+}
+
+segment_list_response = {
+    'data': fields.List(fields.Nested(segment_fields)),
+    'has_more': fields.Boolean,
+    'limit': fields.Integer
+}

api/migrations/versions/2e9819ca5b28_add_tenant_id_in_api_token.py

@@ -0,0 +1,36 @@
+"""add_tenant_id_in_api_token
+
+Revision ID: 2e9819ca5b28
+Revises: 6e2cfb077b04
+Create Date: 2023-09-22 15:41:01.243183
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '2e9819ca5b28'
+down_revision = 'ab23c11305d4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('api_tokens', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('tenant_id', postgresql.UUID(), nullable=True))
+        batch_op.create_index('api_token_tenant_idx', ['tenant_id', 'type'], unique=False)
+        batch_op.drop_column('dataset_id')
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('api_tokens', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('dataset_id', postgresql.UUID(), autoincrement=False, nullable=True))
+        batch_op.drop_index('api_token_tenant_idx')
+        batch_op.drop_column('tenant_id')
+
+    # ### end Alembic commands ###

api/migrations/versions/ab23c11305d4_add_dataset_query_variable_at_app_model_.py

@@ -0,0 +1,31 @@
+"""add dataset query variable at app model configs.
+
+Revision ID: ab23c11305d4
+Revises: 6e2cfb077b04
+Create Date: 2023-09-26 12:22:59.044088
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = 'ab23c11305d4'
+down_revision = '6e2cfb077b04'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('app_model_configs', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('dataset_query_variable', sa.String(length=255), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('app_model_configs', schema=None) as batch_op:
+        batch_op.drop_column('dataset_query_variable')
+
+    # ### end Alembic commands ###

api/models/model.py

@@ -88,6 +88,7 @@ class AppModelConfig(db.Model):
     more_like_this = db.Column(db.Text)
     model = db.Column(db.Text)
     user_input_form = db.Column(db.Text)
+    dataset_query_variable = db.Column(db.String(255))
     pre_prompt = db.Column(db.Text)
     agent_mode = db.Column(db.Text)
     sensitive_word_avoidance = db.Column(db.Text)
@@ -152,6 +153,7 @@ class AppModelConfig(db.Model):
             "sensitive_word_avoidance": self.sensitive_word_avoidance_dict,
             "model": self.model_dict,
             "user_input_form": self.user_input_form_list,
+            "dataset_query_variable": self.dataset_query_variable,
             "pre_prompt": self.pre_prompt,
             "agent_mode": self.agent_mode_dict
         }
@@ -170,6 +172,7 @@ class AppModelConfig(db.Model):
             if model_config.get('sensitive_word_avoidance') else None
         self.model = json.dumps(model_config['model'])
         self.user_input_form = json.dumps(model_config['user_input_form'])
+        self.dataset_query_variable = model_config.get('dataset_query_variable')
         self.pre_prompt = model_config['pre_prompt']
         self.agent_mode = json.dumps(model_config['agent_mode'])
         self.retriever_resource = json.dumps(model_config['retriever_resource']) \
@@ -191,6 +194,7 @@ class AppModelConfig(db.Model):
             sensitive_word_avoidance=self.sensitive_word_avoidance,
             model=self.model,
             user_input_form=self.user_input_form,
+            dataset_query_variable=self.dataset_query_variable,
             pre_prompt=self.pre_prompt,
             agent_mode=self.agent_mode
         )
@@ -625,12 +629,13 @@ class ApiToken(db.Model):
     __table_args__ = (
         db.PrimaryKeyConstraint('id', name='api_token_pkey'),
         db.Index('api_token_app_id_type_idx', 'app_id', 'type'),
-        db.Index('api_token_token_idx', 'token', 'type')
+        db.Index('api_token_token_idx', 'token', 'type'),
+        db.Index('api_token_tenant_idx', 'tenant_id', 'type')
     )
 
     id = db.Column(UUID, server_default=db.text('uuid_generate_v4()'))
     app_id = db.Column(UUID, nullable=True)
-    dataset_id = db.Column(UUID, nullable=True)
+    tenant_id = db.Column(UUID, nullable=True)
     type = db.Column(db.String(16), nullable=False)
     token = db.Column(db.String(255), nullable=False)
     last_used_at = db.Column(db.DateTime, nullable=True)

api/requirements.txt

@@ -51,4 +51,4 @@ stripe~=5.5.0
 pandas==1.5.3
 xinference==0.4.2
 safetensors==0.3.2
-zhipuai==1.0.7
+zhipuai==1.0.7

api/services/account_service.py

@@ -4,11 +4,12 @@ import json
 import logging
 import secrets
 import uuid
-from datetime import datetime
+from datetime import datetime, timedelta
 from hashlib import sha256
 from typing import Optional
 
-from flask import session
+from werkzeug.exceptions import Forbidden, Unauthorized
+from flask import session, current_app
 from sqlalchemy import func
 
 from events.tenant_event import tenant_was_created
@@ -19,16 +20,82 @@ from services.errors.account import AccountLoginError, CurrentPasswordIncorrectE
 from libs.helper import get_remote_ip
 from libs.password import compare_password, hash_password
 from libs.rsa import generate_key_pair
+from libs.passport import PassportService
 from models.account import *
 from tasks.mail_invite_member_task import send_invite_member_mail_task
 
+def _create_tenant_for_account(account):
+    tenant = TenantService.create_tenant(f"{account.name}'s Workspace")
+
+    TenantService.create_tenant_member(tenant, account, role='owner')
+    account.current_tenant = tenant
+
+    return tenant
+
 
 class AccountService:
 
     @staticmethod
-    def load_user(account_id: int) -> Account:
+    def load_user(user_id: str) -> Account:
         # todo: used by flask_login
-        pass
+        if '.' in user_id:
+            tenant_id, account_id = user_id.split('.')
+        else:
+            account_id = user_id
+
+        account = db.session.query(Account).filter(Account.id == account_id).first()
+
+        if account:
+            if account.status == AccountStatus.BANNED.value or account.status == AccountStatus.CLOSED.value:
+                raise Forbidden('Account is banned or closed.')
+
+            workspace_id = session.get('workspace_id')
+            if workspace_id:
+                tenant_account_join = db.session.query(TenantAccountJoin).filter(
+                    TenantAccountJoin.account_id == account.id,
+                    TenantAccountJoin.tenant_id == workspace_id
+                ).first()
+
+                if not tenant_account_join:
+                    tenant_account_join = db.session.query(TenantAccountJoin).filter(
+                        TenantAccountJoin.account_id == account.id).first()
+
+                    if tenant_account_join:
+                        account.current_tenant_id = tenant_account_join.tenant_id
+                    else:
+                        _create_tenant_for_account(account)
+                    session['workspace_id'] = account.current_tenant_id
+                else:
+                    account.current_tenant_id = workspace_id
+            else:
+                tenant_account_join = db.session.query(TenantAccountJoin).filter(
+                    TenantAccountJoin.account_id == account.id).first()
+                if tenant_account_join:
+                    account.current_tenant_id = tenant_account_join.tenant_id
+                else:
+                    _create_tenant_for_account(account)
+                session['workspace_id'] = account.current_tenant_id
+
+            current_time = datetime.utcnow()
+
+            # update last_active_at when last_active_at is more than 10 minutes ago
+            if current_time - account.last_active_at > timedelta(minutes=10):
+                account.last_active_at = current_time
+                db.session.commit()
+
+        return account
+    
+    @staticmethod
+    def get_account_jwt_token(account):
+        payload = {
+            "user_id": account.id,
+            "exp": datetime.utcnow() + timedelta(days=30),
+            "iss":  current_app.config['EDITION'],
+            "sub": 'Console API Passport',
+        }
+
+        token = PassportService().issue(payload)
+        return token
 
     @staticmethod
     def authenticate(email: str, password: str) -> Account:

api/services/app_model_config_service.py

@@ -81,7 +81,7 @@ class AppModelConfigService:
         return filtered_cp
 
     @staticmethod
-    def validate_configuration(tenant_id: str, account: Account, config: dict) -> dict:
+    def validate_configuration(tenant_id: str, account: Account, config: dict, mode: str) -> dict:
         # opening_statement
         if 'opening_statement' not in config or not config["opening_statement"]:
             config["opening_statement"] = ""
@@ -335,6 +335,9 @@ class AppModelConfigService:
 
                 if not AppModelConfigService.is_dataset_exists(account, tool_item["id"]):
                     raise ValueError("Dataset ID does not exist, please check your permission.")
+        
+        # dataset_query_variable
+        AppModelConfigService.is_dataset_query_variable_valid(config, mode)
 
         # Filter out extra parameters
         filtered_config = {
@@ -351,8 +354,25 @@ class AppModelConfigService:
                 "completion_params": config["model"]["completion_params"]
             },
             "user_input_form": config["user_input_form"],
+            "dataset_query_variable": config.get('dataset_query_variable'),
             "pre_prompt": config["pre_prompt"],
             "agent_mode": config["agent_mode"]
         }
 
         return filtered_config
+    
+    @staticmethod
+    def is_dataset_query_variable_valid(config: dict, mode: str) -> None:
+        # Only check when mode is completion
+        if mode != 'completion':
+            return
+        
+        agent_mode = config.get("agent_mode", {})
+        tools = agent_mode.get("tools", [])
+        dataset_exists = "dataset" in str(tools)
+        
+        dataset_query_variable = config.get("dataset_query_variable")
+
+        if dataset_exists and not dataset_query_variable:
+            raise ValueError("Dataset query variable is required when dataset is exist")
+

api/services/completion_service.py

@@ -117,7 +117,8 @@ class CompletionService:
                 model_config = AppModelConfigService.validate_configuration(
                     tenant_id=app_model.tenant_id,
                     account=user,
-                    config=args['model_config']
+                    config=args['model_config'],
+                    mode=app_model.mode
                 )
 
                 app_model_config = AppModelConfig(

api/services/dataset_service.py

@@ -96,7 +96,7 @@ class DatasetService:
         embedding_model = None
         if indexing_technique == 'high_quality':
             embedding_model = ModelFactory.get_embedding_model(
-                tenant_id=current_user.current_tenant_id
+                tenant_id=tenant_id
             )
         dataset = Dataset(name=name, indexing_technique=indexing_technique)
         # dataset = Dataset(name=name, provider=provider, config=config)
@@ -477,6 +477,7 @@ class DocumentService:
                 )
                 dataset.collection_binding_id = dataset_collection_binding.id
 
+
         documents = []
         batch = time.strftime('%Y%m%d%H%M%S') + str(random.randint(100000, 999999))
         if 'original_document_id' in document_data and document_data["original_document_id"]:
@@ -626,6 +627,9 @@ class DocumentService:
         document = DocumentService.get_document(dataset.id, document_data["original_document_id"])
         if document.display_status != 'available':
             raise ValueError("Document is not available")
+        # update document name
+        if 'name' in document_data and document_data['name']:
+            document.name = document_data['name']
         # save process rule
         if 'process_rule' in document_data and document_data['process_rule']:
             process_rule = document_data["process_rule"]
@@ -767,7 +771,7 @@ class DocumentService:
         return dataset, documents, batch
 
     @classmethod
-    def document_create_args_validate(cls, args: dict):
+    def  document_create_args_validate(cls, args: dict):
         if 'original_document_id' not in args or not args['original_document_id']:
             DocumentService.data_source_args_validate(args)
             DocumentService.process_rule_args_validate(args)
@@ -1014,6 +1018,66 @@ class SegmentService:
         segment = db.session.query(DocumentSegment).filter(DocumentSegment.id == segment_document.id).first()
         return segment
 
+    @classmethod
+    def multi_create_segment(cls, segments: list, document: Document, dataset: Dataset):
+        embedding_model = None
+        if dataset.indexing_technique == 'high_quality':
+            embedding_model = ModelFactory.get_embedding_model(
+                tenant_id=dataset.tenant_id,
+                model_provider_name=dataset.embedding_model_provider,
+                model_name=dataset.embedding_model
+            )
+        max_position = db.session.query(func.max(DocumentSegment.position)).filter(
+            DocumentSegment.document_id == document.id
+        ).scalar()
+        pre_segment_data_list = []
+        segment_data_list = []
+        for segment_item in segments:
+            content = segment_item['content']
+            doc_id = str(uuid.uuid4())
+            segment_hash = helper.generate_text_hash(content)
+            tokens = 0
+            if dataset.indexing_technique == 'high_quality' and embedding_model:
+                # calc embedding use tokens
+                tokens = embedding_model.get_num_tokens(content)
+            segment_document = DocumentSegment(
+                tenant_id=current_user.current_tenant_id,
+                dataset_id=document.dataset_id,
+                document_id=document.id,
+                index_node_id=doc_id,
+                index_node_hash=segment_hash,
+                position=max_position + 1 if max_position else 1,
+                content=content,
+                word_count=len(content),
+                tokens=tokens,
+                status='completed',
+                indexing_at=datetime.datetime.utcnow(),
+                completed_at=datetime.datetime.utcnow(),
+                created_by=current_user.id
+            )
+            if document.doc_form == 'qa_model':
+                segment_document.answer = segment_item['answer']
+            db.session.add(segment_document)
+            segment_data_list.append(segment_document)
+            pre_segment_data = {
+                'segment': segment_document,
+                'keywords': segment_item['keywords']
+            }
+            pre_segment_data_list.append(pre_segment_data)
+
+        try:
+            # save vector index
+            VectorService.multi_create_segment_vector(pre_segment_data_list, dataset)
+        except Exception as e:
+            logging.exception("create segment index failed")
+            for segment_document in segment_data_list:
+                segment_document.enabled = False
+                segment_document.disabled_at = datetime.datetime.utcnow()
+                segment_document.status = 'error'
+                segment_document.error = str(e)
+        db.session.commit()
+        return segment_data_list
+
     @classmethod
     def update_segment(cls, args: dict, segment: DocumentSegment, document: Document, dataset: Dataset):
         indexing_cache_key = 'segment_{}_indexing'.format(segment.id)

api/services/errors/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding:utf-8 -*-
 __all__ = [
     'base', 'conversation', 'message', 'index', 'app_model_config', 'account', 'document', 'dataset',
-    'app', 'completion', 'audio'
+    'app', 'completion', 'audio', 'file'
 ]
 
 from . import *

api/services/errors/file.py

@@ -3,3 +3,11 @@ from services.errors.base import BaseServiceError
 
 class FileNotExistsError(BaseServiceError):
     pass
+
+
+class FileTooLargeError(BaseServiceError):
+    description = "{message}"
+
+
+class UnsupportedFileTypeError(BaseServiceError):
+    pass

api/services/file_service.py

@@ -0,0 +1,123 @@
+import datetime
+import hashlib
+import time
+import uuid
+
+from cachetools import TTLCache
+from flask import request, current_app
+from flask_login import current_user
+from werkzeug.datastructures import FileStorage
+from werkzeug.exceptions import NotFound
+
+from core.data_loader.file_extractor import FileExtractor
+from extensions.ext_storage import storage
+from extensions.ext_database import db
+from models.model import UploadFile
+from services.errors.file import FileTooLargeError, UnsupportedFileTypeError
+
+ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx', 'docx', 'csv']
+PREVIEW_WORDS_LIMIT = 3000
+cache = TTLCache(maxsize=None, ttl=30)
+
+
+class FileService:
+
+    @staticmethod
+    def upload_file(file: FileStorage) -> UploadFile:
+        # read file content
+        file_content = file.read()
+        # get file size
+        file_size = len(file_content)
+
+        file_size_limit = current_app.config.get("UPLOAD_FILE_SIZE_LIMIT") * 1024 * 1024
+        if file_size > file_size_limit:
+            message = f'File size exceeded. {file_size} > {file_size_limit}'
+            raise FileTooLargeError(message)
+
+        extension = file.filename.split('.')[-1]
+        if extension.lower() not in ALLOWED_EXTENSIONS:
+            raise UnsupportedFileTypeError()
+
+        # user uuid as file name
+        file_uuid = str(uuid.uuid4())
+        file_key = 'upload_files/' + current_user.current_tenant_id + '/' + file_uuid + '.' + extension
+
+        # save file to storage
+        storage.save(file_key, file_content)
+
+        # save file to db
+        config = current_app.config
+        upload_file = UploadFile(
+            tenant_id=current_user.current_tenant_id,
+            storage_type=config['STORAGE_TYPE'],
+            key=file_key,
+            name=file.filename,
+            size=file_size,
+            extension=extension,
+            mime_type=file.mimetype,
+            created_by=current_user.id,
+            created_at=datetime.datetime.utcnow(),
+            used=False,
+            hash=hashlib.sha3_256(file_content).hexdigest()
+        )
+
+        db.session.add(upload_file)
+        db.session.commit()
+
+        return upload_file
+
+    @staticmethod
+    def upload_text(text: str, text_name: str) -> UploadFile:
+        # user uuid as file name
+        file_uuid = str(uuid.uuid4())
+        file_key = 'upload_files/' + current_user.current_tenant_id + '/' + file_uuid + '.txt'
+
+        # save file to storage
+        storage.save(file_key, text.encode('utf-8'))
+
+        # save file to db
+        config = current_app.config
+        upload_file = UploadFile(
+            tenant_id=current_user.current_tenant_id,
+            storage_type=config['STORAGE_TYPE'],
+            key=file_key,
+            name=text_name + '.txt',
+            size=len(text),
+            extension='txt',
+            mime_type='text/plain',
+            created_by=current_user.id,
+            created_at=datetime.datetime.utcnow(),
+            used=True,
+            used_by=current_user.id,
+            used_at=datetime.datetime.utcnow()
+        )
+
+        db.session.add(upload_file)
+        db.session.commit()
+
+        return upload_file
+
+    @staticmethod
+    def get_file_preview(file_id: str) -> str:
+        # get file storage key
+        key = file_id + request.path
+        cached_response = cache.get(key)
+        if cached_response and time.time() - cached_response['timestamp'] < cache.ttl:
+            return cached_response['response']
+
+        upload_file = db.session.query(UploadFile) \
+            .filter(UploadFile.id == file_id) \
+            .first()
+
+        if not upload_file:
+            raise NotFound("File not found")
+
+        # extract text from file
+        extension = upload_file.extension
+        if extension.lower() not in ALLOWED_EXTENSIONS:
+            raise UnsupportedFileTypeError()
+
+        text = FileExtractor.load(upload_file, return_text=True)
+        text = text[0:PREVIEW_WORDS_LIMIT] if text else ''
+
+        return text

api/services/vector_service.py

@@ -35,6 +35,32 @@ class VectorService:
             else:
                 index.add_texts([document])
 
+    @classmethod
+    def multi_create_segment_vector(cls, pre_segment_data_list: list, dataset: Dataset):
+        documents = []
+        for pre_segment_data in pre_segment_data_list:
+            segment = pre_segment_data['segment']
+            document = Document(
+                page_content=segment.content,
+                metadata={
+                    "doc_id": segment.index_node_id,
+                    "doc_hash": segment.index_node_hash,
+                    "document_id": segment.document_id,
+                    "dataset_id": segment.dataset_id,
+                }
+            )
+            documents.append(document)
+
+        # save vector index
+        index = IndexBuilder.get_index(dataset, 'high_quality')
+        if index:
+            index.add_texts(documents, duplicate_check=True)
+
+        # save keyword index
+        keyword_index = IndexBuilder.get_index(dataset, 'economy')
+        if keyword_index:
+            keyword_index.multi_create_segment_keywords(pre_segment_data_list)
+
     @classmethod
     def update_segment_vector(cls, keywords: Optional[List[str]], segment: DocumentSegment, dataset: Dataset):
         # update segment index task

api/tasks/clean_dataset_task.py

@@ -13,13 +13,15 @@ from models.dataset import DocumentSegment, Dataset, DatasetKeywordTable, Datase
 
 
 @shared_task(queue='dataset')
-def clean_dataset_task(dataset_id: str, tenant_id: str, indexing_technique: str, index_struct: str):
+def clean_dataset_task(dataset_id: str, tenant_id: str, indexing_technique: str,
+                       index_struct: str, collection_binding_id: str):
     """
     Clean dataset when dataset deleted.
     :param dataset_id: dataset id
     :param tenant_id: tenant id
     :param indexing_technique: indexing technique
     :param index_struct: index struct dict
+    :param collection_binding_id: collection binding id
 
     Usage: clean_dataset_task.delay(dataset_id, tenant_id, indexing_technique, index_struct)
     """
@@ -31,9 +33,9 @@ def clean_dataset_task(dataset_id: str, tenant_id: str, indexing_technique: str,
             id=dataset_id,
             tenant_id=tenant_id,
             indexing_technique=indexing_technique,
-            index_struct=index_struct
+            index_struct=index_struct,
+            collection_binding_id=collection_binding_id
         )
-
         documents = db.session.query(Document).filter(Document.dataset_id == dataset_id).all()
         segments = db.session.query(DocumentSegment).filter(DocumentSegment.dataset_id == dataset_id).all()
 
@@ -43,7 +45,7 @@ def clean_dataset_task(dataset_id: str, tenant_id: str, indexing_technique: str,
         if dataset.indexing_technique == 'high_quality':
             vector_index = IndexBuilder.get_default_high_quality_index(dataset)
             try:
-                vector_index.delete()
+                vector_index.delete_by_group_id(dataset.id)
             except Exception:
                 logging.exception("Delete doc index failed when dataset deleted.")
 

api/tasks/deal_dataset_vector_index_task.py

@@ -31,8 +31,8 @@ def deal_dataset_vector_index_task(dataset_id: str, action: str):
             raise Exception('Dataset not found')
 
         if action == "remove":
-            index = IndexBuilder.get_index(dataset, 'high_quality', ignore_high_quality_check=False)
-            index.delete()
+            index = IndexBuilder.get_index(dataset, 'high_quality', ignore_high_quality_check=True)
+            index.delete_by_group_id(dataset.id)
         elif action == "add":
             dataset_documents = db.session.query(DatasetDocument).filter(
                 DatasetDocument.dataset_id == dataset_id,

api/tests/integration_tests/.env.example

@@ -14,6 +14,7 @@ REPLICATE_API_TOKEN=
 # Hugging Face API Key
 HUGGINGFACE_API_KEY=
 HUGGINGFACE_ENDPOINT_URL=
+HUGGINGFACE_EMBEDDINGS_ENDPOINT_URL=
 
 # Minimax Credentials
 MINIMAX_API_KEY=

api/tests/integration_tests/models/embedding/test_huggingface_hub_embedding.py

@@ -0,0 +1,136 @@
+import json
+import os
+from unittest.mock import patch, MagicMock
+
+from core.model_providers.models.entity.model_params import ModelType
+from core.model_providers.models.embedding.huggingface_embedding import HuggingfaceEmbedding
+from core.model_providers.providers.huggingface_hub_provider import HuggingfaceHubProvider
+from models.provider import Provider, ProviderType, ProviderModel
+
+DEFAULT_MODEL_NAME = 'obrizum/all-MiniLM-L6-v2'
+
+def get_mock_provider():
+    return Provider(
+        id='provider_id',
+        tenant_id='tenant_id',
+        provider_name='huggingface_hub',
+        provider_type=ProviderType.CUSTOM.value,
+        encrypted_config='',
+        is_valid=True,
+    )
+
+
+def get_mock_embedding_model(model_name, huggingfacehub_api_type, mocker):
+    valid_api_key = os.environ['HUGGINGFACE_API_KEY']
+    endpoint_url = os.environ['HUGGINGFACE_EMBEDDINGS_ENDPOINT_URL']
+    model_provider = HuggingfaceHubProvider(provider=get_mock_provider())
+
+    credentials = {
+        'huggingfacehub_api_type': huggingfacehub_api_type,
+        'huggingfacehub_api_token': valid_api_key,
+        'task_type': 'feature-extraction'
+    }
+
+    if huggingfacehub_api_type == 'inference_endpoints':
+        credentials['huggingfacehub_endpoint_url'] = endpoint_url
+
+    mock_query = MagicMock()
+    mock_query.filter.return_value.first.return_value = ProviderModel(
+        provider_name='huggingface_hub',
+        model_name=model_name,
+        model_type=ModelType.EMBEDDINGS.value,
+        encrypted_config=json.dumps(credentials),
+        is_valid=True,
+    )
+    mocker.patch('extensions.ext_database.db.session.query',
+                 return_value=mock_query)
+
+    return HuggingfaceEmbedding(
+        model_provider=model_provider,
+        name=model_name
+    )
+
+
+def decrypt_side_effect(tenant_id, encrypted_api_key):
+    return encrypted_api_key
+
+
+@patch('core.helper.encrypter.decrypt_token', side_effect=decrypt_side_effect)
+def test_hosted_inference_api_embed_documents(mock_decrypt, mocker):
+    embedding_model = get_mock_embedding_model(
+        DEFAULT_MODEL_NAME,
+        'hosted_inference_api',
+        mocker)
+    rst = embedding_model.client.embed_documents(['test', 'test1'])
+    assert isinstance(rst, list)
+    assert len(rst) == 2
+    assert len(rst[0]) == 384
+
+
+@patch('core.helper.encrypter.decrypt_token', side_effect=decrypt_side_effect)
+def test_endpoint_url_inference_api_embed_documents(mock_decrypt, mocker):
+    embedding_model = get_mock_embedding_model(
+        '',
+        'inference_endpoints',
+        mocker)
+    mocker.patch('core.third_party.langchain.embeddings.huggingface_hub_embedding.InferenceClient.post'
+                 , return_value=bytes(json.dumps([[1, 2, 3], [4, 5, 6]]), 'utf-8'))
+    
+    rst = embedding_model.client.embed_documents(['test', 'test1'])
+    assert isinstance(rst, list)
+    assert len(rst) == 2
+    assert len(rst[0]) == 3
+
+@patch('core.helper.encrypter.decrypt_token', side_effect=decrypt_side_effect)
+def test_endpoint_url_inference_api_embed_documents_two(mock_decrypt, mocker):
+    embedding_model = get_mock_embedding_model(
+        '',
+        'inference_endpoints',
+        mocker)
+    mocker.patch('core.third_party.langchain.embeddings.huggingface_hub_embedding.InferenceClient.post'
+                 , return_value=bytes(json.dumps([[[[1,2,3],[4,5,6],[7,8,9]]],[[[1,2,3],[4,5,6],[7,8,9]]]]), 'utf-8'))
+    
+    rst = embedding_model.client.embed_documents(['test', 'test1'])
+    assert isinstance(rst, list)
+    assert len(rst) == 2
+    assert len(rst[0]) == 3
+
+
+@patch('core.helper.encrypter.decrypt_token', side_effect=decrypt_side_effect)
+def test_hosted_inference_api_embed_query(mock_decrypt, mocker):
+    embedding_model = get_mock_embedding_model(
+        DEFAULT_MODEL_NAME,
+        'hosted_inference_api',
+        mocker)
+    rst = embedding_model.client.embed_query('test')
+    assert isinstance(rst, list)
+    assert len(rst) == 384
+
+
+@patch('core.helper.encrypter.decrypt_token', side_effect=decrypt_side_effect)
+def test_endpoint_url_inference_api_embed_query(mock_decrypt, mocker):
+    embedding_model = get_mock_embedding_model(
+        '',
+        'inference_endpoints',
+        mocker)
+    
+    mocker.patch('core.third_party.langchain.embeddings.huggingface_hub_embedding.InferenceClient.post'
+                 , return_value=bytes(json.dumps([[1, 2, 3]]), 'utf-8'))
+
+    rst = embedding_model.client.embed_query('test')
+    assert isinstance(rst, list)
+    assert len(rst) == 3
+
+@patch('core.helper.encrypter.decrypt_token', side_effect=decrypt_side_effect)
+def test_endpoint_url_inference_api_embed_query_two(mock_decrypt, mocker):
+    embedding_model = get_mock_embedding_model(
+        '',
+        'inference_endpoints',
+        mocker)
+    
+    mocker.patch('core.third_party.langchain.embeddings.huggingface_hub_embedding.InferenceClient.post'
+                 , return_value=bytes(json.dumps([[[[1,2,3],[4,5,6],[7,8,9]]]]), 'utf-8'))
+
+    rst = embedding_model.client.embed_query('test')
+    assert isinstance(rst, list)
+    assert len(rst) == 3

docker/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.3.23
+    image: langgenius/dify-api:0.3.24
     restart: always
     environment:
       # Startup mode, 'api' starts the API server.
@@ -49,15 +49,6 @@ services:
       REDIS_USE_SSL: 'false'
       # use redis db 0 for redis cache
       REDIS_DB: 0
-      # The configurations of session, Supported values are `sqlalchemy`. `redis`
-      SESSION_TYPE: redis
-      SESSION_REDIS_HOST: redis
-      SESSION_REDIS_PORT: 6379
-      SESSION_REDIS_USERNAME: ''
-      SESSION_REDIS_PASSWORD: difyai123456
-      SESSION_REDIS_USE_SSL: 'false'
-      # use redis db 2 for session store
-      SESSION_REDIS_DB: 2
       # The configurations of celery broker.
       # Use redis as the broker, and redis db 1 for celery broker.
       CELERY_BROKER_URL: redis://:difyai123456@redis:6379/1
@@ -76,10 +67,6 @@ services:
       # If you want to enable cross-origin support,
       # you must use the HTTPS protocol and set the configuration to `SameSite=None, Secure=true, HttpOnly=true`.
       #
-      # For **production** purposes, please set `SameSite=Lax, Secure=true, HttpOnly=true`.
-      COOKIE_HTTPONLY: 'true'
-      COOKIE_SAMESITE: 'Lax'
-      COOKIE_SECURE: 'false'
       # The type of storage to use for storing user files. Supported values are `local` and `s3`, Default: `local`
       STORAGE_TYPE: local
       # The path to the local storage directory, the directory relative the root path of API service codes or absolute path. Default: `storage` or `/home/john/storage`.
@@ -124,7 +111,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.3.23
+    image: langgenius/dify-api:0.3.24
     restart: always
     environment:
       # Startup mode, 'worker' starts the Celery worker for processing the queue.
@@ -176,7 +163,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.3.23
+    image: langgenius/dify-web:0.3.24
     restart: always
     environment:
       EDITION: SELF_HOSTED
@@ -196,6 +183,7 @@ services:
     image: postgres:15-alpine
     restart: always
     environment:
+      PGUSER: postgres
       # The password for the default postgres user.
       POSTGRES_PASSWORD: difyai123456
       # The name of the default postgres database.

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx

@@ -38,9 +38,9 @@ const CardView: FC<ICardViewProps> = ({ appId }) => {
 
     message ||= (type === 'success' ? 'modifiedSuccessfully' : 'modifiedUnsuccessfully')
 
-    if (type === 'success') {
+    if (type === 'success')
       mutate(detailParams)
-    }
+
     notify({
       type,
       message: t(`common.actionMsg.${message}`),
@@ -79,7 +79,7 @@ const CardView: FC<ICardViewProps> = ({ appId }) => {
     if (!err)
       localStorage.setItem(NEED_REFRESH_APP_LIST_KEY, '1')
 
-      handleCallbackResult(err)
+    handleCallbackResult(err)
   }
 
   const onGenerateCode = async () => {

web/app/(commonLayout)/apps/AppCard.tsx

@@ -64,8 +64,8 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => {
 
   const getAppDetail = async () => {
     setDetailState({ loading: true })
-    const [err, res] = await asyncRunSafe<App>(
-      fetchAppDetail({ url: '/apps', id: app.id }) as Promise<App>,
+    const [err, res] = await asyncRunSafe(
+      fetchAppDetail({ url: '/apps', id: app.id }),
     )
     if (!err) {
       setDetailState({ loading: false, detail: res })
@@ -76,11 +76,11 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => {
 
   const onSaveSiteConfig = useCallback(
     async (params: ConfigParams) => {
-      const [err] = await asyncRunSafe<App>(
+      const [err] = await asyncRunSafe(
         updateAppSiteConfig({
           url: `/apps/${app.id}/site`,
           body: params,
-        }) as Promise<App>,
+        }),
       )
       if (!err) {
         notify({

web/app/(commonLayout)/datasets/(datasetDetailLayout)/[datasetId]/layout.tsx

@@ -19,7 +19,7 @@ import {
 } from '@heroicons/react/24/solid'
 import Link from 'next/link'
 import s from './style.module.css'
-import { fetchDataDetail, fetchDatasetRelatedApps } from '@/service/datasets'
+import { fetchDatasetDetail, fetchDatasetRelatedApps } from '@/service/datasets'
 import type { RelatedApp } from '@/models/datasets'
 import { getLocaleOnClient } from '@/i18n/client'
 import AppSideBar from '@/app/components/app-sidebar'
@@ -30,8 +30,6 @@ import Loading from '@/app/components/base/loading'
 import DatasetDetailContext from '@/context/dataset-detail'
 import { DataSourceType } from '@/models/datasets'
 
-// import { fetchDatasetDetail } from '@/service/datasets'
-
 export type IAppDetailLayoutProps = {
   children: React.ReactNode
   params: { datasetId: string }
@@ -94,9 +92,9 @@ const DatasetDetailLayout: FC<IAppDetailLayoutProps> = (props) => {
   const hideSideBar = /documents\/create$/.test(pathname)
   const { t } = useTranslation()
   const { data: datasetRes, error, mutate: mutateDatasetRes } = useSWR({
-    action: 'fetchDataDetail',
+    url: 'fetchDatasetDetail',
     datasetId,
-  }, apiParams => fetchDataDetail(apiParams.datasetId))
+  }, apiParams => fetchDatasetDetail(apiParams.datasetId))
 
   const { data: relatedApps } = useSWR({
     action: 'fetchDatasetRelatedApps',

web/app/(commonLayout)/datasets/ApiServer.tsx

@@ -0,0 +1,41 @@
+'use client'
+
+import type { FC } from 'react'
+import { useTranslation } from 'react-i18next'
+import CopyFeedback from '@/app/components/base/copy-feedback'
+import SecretKeyButton from '@/app/components/develop/secret-key/secret-key-button'
+import { randomString } from '@/utils'
+
+type ApiServerProps = {
+  apiBaseUrl: string
+}
+const ApiServer: FC<ApiServerProps> = ({
+  apiBaseUrl,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <div className='flex items-center'>
+      <div className='flex items-center mr-2 pl-1.5 pr-1 h-8 bg-white/80 border-[0.5px] border-white rounded-lg'>
+        <div className='mr-0.5 px-1.5 h-5 border border-gray-200 text-[11px] text-gray-500 rounded-md'>{t('appApi.apiServer')}</div>
+        <div className='px-1 w-[248px] text-[13px] font-medium text-gray-800'>{apiBaseUrl}</div>
+        <div className='mx-1 w-[1px] h-[14px] bg-gray-200'></div>
+        <CopyFeedback
+          content={apiBaseUrl}
+          selectorId={randomString(8)}
+          className={'!w-6 !h-6 hover:bg-gray-200'}
+        />
+      </div>
+      <div className='flex items-center mr-2 px-3 h-8 bg-[#ECFDF3] text-xs font-semibold text-[#039855] rounded-lg border-[0.5px] border-[#D1FADF]'>
+        {t('appApi.ok')}
+      </div>
+      <SecretKeyButton
+        className='flex-shrink-0 !h-8 bg-white'
+        textCls='!text-gray-700 font-medium'
+        iconCls='stroke-[1.2px]'
+      />
+    </div>
+  )
+}
+
+export default ApiServer

web/app/(commonLayout)/datasets/Container.tsx

@@ -0,0 +1,56 @@
+'use client'
+
+// Libraries
+import { useRef, useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import useSWR from 'swr'
+
+// Components
+import Datasets from './Datasets'
+import DatasetFooter from './DatasetFooter'
+import ApiServer from './ApiServer'
+import Doc from './Doc'
+import TabSlider from '@/app/components/base/tab-slider'
+
+// Services
+import { fetchDatasetApiBaseUrl } from '@/service/datasets'
+
+const Container = () => {
+  const { t } = useTranslation()
+
+  const options = [
+    { value: 'dataset', text: t('dataset.datasets') },
+    { value: 'api', text: t('dataset.datasetsApi') },
+  ]
+
+  const [activeTab, setActiveTab] = useState('dataset')
+  const containerRef = useRef<HTMLDivElement>(null)
+  const { data } = useSWR(activeTab === 'dataset' ? null : '/datasets/api-base-info', fetchDatasetApiBaseUrl)
+
+  return (
+    <div ref={containerRef} className='grow relative flex flex-col bg-gray-100 overflow-y-auto'>
+      <div className='sticky top-0 flex justify-between pt-4 px-12 pb-2 h-14 bg-gray-100 z-10'>
+        <TabSlider
+          value={activeTab}
+          onChange={newActiveTab => setActiveTab(newActiveTab)}
+          options={options}
+        />
+        {activeTab === 'api' && data && <ApiServer apiBaseUrl={data.api_base_url || ''} />}
+      </div>
+
+      {activeTab === 'dataset'
+        ? (
+          <>
+            <Datasets containerRef={containerRef} />
+            <DatasetFooter />
+          </>
+        )
+        : (
+          activeTab === 'api' && data && <Doc apiBaseUrl={data.api_base_url || ''} />
+        )}
+    </div>
+
+  )
+}
+
+export default Container

web/app/(commonLayout)/datasets/Datasets.tsx

@@ -7,7 +7,7 @@ import NewDatasetCard from './NewDatasetCard'
 import DatasetCard from './DatasetCard'
 import type { DataSetListResponse } from '@/models/datasets'
 import { fetchDatasets } from '@/service/datasets'
-import { useAppContext, useSelector } from '@/context/app-context'
+import { useAppContext } from '@/context/app-context'
 
 const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
   if (!pageIndex || previousPageData.has_more)
@@ -15,11 +15,16 @@ const getKey = (pageIndex: number, previousPageData: DataSetListResponse) => {
   return null
 }
 
-const Datasets = () => {
+type Props = {
+  containerRef: React.RefObject<HTMLDivElement>
+}
+
+const Datasets = ({
+  containerRef,
+}: Props) => {
   const { isCurrentWorkspaceManager } = useAppContext()
-  const { data, isLoading, setSize, mutate } = useSWRInfinite(getKey, fetchDatasets, { revalidateFirstPage: false })
+  const { data, isLoading, setSize, mutate } = useSWRInfinite(getKey, fetchDatasets, { revalidateFirstPage: false, revalidateAll: true })
   const loadingStateRef = useRef(false)
-  const pageContainerRef = useSelector(state => state.pageContainerRef)
   const anchorRef = useRef<HTMLAnchorElement>(null)
 
   useEffect(() => {
@@ -29,19 +34,19 @@ const Datasets = () => {
   useEffect(() => {
     const onScroll = debounce(() => {
       if (!loadingStateRef.current) {
-        const { scrollTop, clientHeight } = pageContainerRef.current!
+        const { scrollTop, clientHeight } = containerRef.current!
         const anchorOffset = anchorRef.current!.offsetTop
         if (anchorOffset - scrollTop - clientHeight < 100)
           setSize(size => size + 1)
       }
     }, 50)
 
-    pageContainerRef.current?.addEventListener('scroll', onScroll)
-    return () => pageContainerRef.current?.removeEventListener('scroll', onScroll)
+    containerRef.current?.addEventListener('scroll', onScroll)
+    return () => containerRef.current?.removeEventListener('scroll', onScroll)
   }, [])
 
   return (
-    <nav className='grid content-start grid-cols-1 gap-4 px-12 pt-8 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 grow shrink-0'>
+    <nav className='grid content-start grid-cols-1 gap-4 px-12 pt-2 sm:grid-cols-2 md:grid-cols-3 lg:grid-cols-4 grow shrink-0'>
       { isCurrentWorkspaceManager && <NewDatasetCard ref={anchorRef} /> }
       {data?.map(({ data: datasets }) => datasets.map(dataset => (
         <DatasetCard key={dataset.id} dataset={dataset} onDelete={mutate} />),

web/app/(commonLayout)/datasets/Doc.tsx

@@ -0,0 +1,28 @@
+'use client'
+
+import type { FC } from 'react'
+import { useContext } from 'use-context-selector'
+import TemplateEn from './template/template.en.mdx'
+import TemplateZh from './template/template.zh.mdx'
+import I18n from '@/context/i18n'
+
+type DocProps = {
+  apiBaseUrl: string
+}
+const Doc: FC<DocProps> = ({
+  apiBaseUrl,
+}) => {
+  const { locale } = useContext(I18n)
+
+  return (
+    <article className='mx-12 pt-16 bg-white rounded-t-xl prose prose-xl'>
+      {
+        locale === 'en'
+          ? <TemplateEn apiBaseUrl={apiBaseUrl} />
+          : <TemplateZh apiBaseUrl={apiBaseUrl} />
+      }
+    </article>
+  )
+}
+
+export default Doc

web/app/(commonLayout)/datasets/page.tsx

@@ -1,12 +1,8 @@
-import Datasets from './Datasets'
-import DatasetFooter from './DatasetFooter'
+import Container from './Container'
 
 const AppList = async () => {
   return (
-    <div className='flex flex-col overflow-auto bg-gray-100 shrink-0 grow'>
-      <Datasets />
-      <DatasetFooter />
-    </div >
+    <Container />
   )
 }
 

web/app/(commonLayout)/datasets/template/template.en.mdx

@@ -0,0 +1,883 @@
+import { CodeGroup } from '@/app/components/develop/code.tsx'
+import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from '@/app/components/develop/md.tsx'
+
+# Dataset API
+<br/>
+<br/>
+<Heading
+  url='/datasets'
+  method='POST'
+  title='Create an empty dataset'
+  name='#create_empty_dataset'
+/>
+<Row>
+  <Col>
+    ### Request Body
+    <Properties>
+      <Property name='name' type='string' key='name'>
+        Dataset name
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets"
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name"}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request POST '${apiBaseUrl}/v1/datasets' \
+    --header 'Authorization: Bearer {api_key}' \
+    --header 'Content-Type: application/json' \
+    --data-raw '{
+      "name": "name"
+    }'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "id": "",
+      "name": "name",
+      "description": null,
+      "provider": "vendor",
+      "permission": "only_me",
+      "data_source_type": null,
+      "indexing_technique": null,
+      "app_count": 0,
+      "document_count": 0,
+      "word_count": 0,
+      "created_by": "",
+      "created_at": 1695636173,
+      "updated_by": "",
+      "updated_at": 1695636173,
+      "embedding_model": null,
+      "embedding_model_provider": null,
+      "embedding_available": null
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets'
+  method='GET'
+  title='Dataset list'
+  name='#dataset_list'
+/>
+<Row>
+  <Col>
+    ### Query
+    <Properties>
+      <Property name='page' type='string' key='page'>
+        Page number
+      </Property>
+      <Property name='limit' type='string' key='limit'>
+        Number of items returned, default 20, range 1-100
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets"
+      targetCode={`curl --location --request GET '${props.apiBaseUrl}/datasets?page=1&limit=20' \\\n--header 'Authorization: Bearer {api_key}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request GET 'https://api.dify.ai/v1/datasets?page=1&limit=20' \
+    --header 'Authorization: Bearer {api_key}'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "data": [
+        {
+          "id": "",
+          "name": "name",
+          "description": "desc",
+          "permission": "only_me",
+          "data_source_type": "upload_file",
+          "indexing_technique": "",
+          "app_count": 2,
+          "document_count": 10,
+          "word_count": 1200,
+          "created_by": "",
+          "created_at": "",
+          "updated_by": "",
+          "updated_at": ""
+        },
+        ...
+      ],
+      "has_more": true,
+      "limit": 20,
+      "total": 50,
+      "page": 1
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/document/create_by_text'
+  method='POST'
+  title='Create a document from text'
+  name='#create_by_text'
+/>
+<Row>
+  <Col>
+    This api is based on an existing dataset and creates a new document through text based on this dataset.
+
+    ### Params
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        Dataset ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='name' type='string' key='name'>
+        Document name
+      </Property>
+      <Property name='text' type='string' key='text'>
+        Document content
+      </Property>
+      <Property name='indexing_technique' type='string' key='indexing_technique'>
+        Index mode
+          - <code>high_quality</code> High quality: embedding using embedding model, built as vector database index
+          - <code>economy</code> Economy: Build using inverted index of Keyword Table Index
+      </Property>
+      <Property name='process_rule' type='object' key='process_rule'>
+        Processing rules
+          - <code>mode</code> (string) Cleaning, segmentation mode, automatic / custom
+          - <code>rules</code> (object) Custom rules (in automatic mode, this field is empty)
+            - <code>pre_processing_rules</code> (array[object]) Preprocessing rules
+              - <code>id</code> (string) Unique identifier for the preprocessing rule
+                - enumerate 
+                  - <code>remove_extra_spaces</code> Replace consecutive spaces, newlines, tabs
+                  - <code>remove_urls_emails</code> Delete URL, email address
+              - <code>enabled</code> (bool) Whether to select this rule or not. If no document ID is passed in, it represents the default value.
+            - <code>segmentation</code> (object) segmentation rules
+              - <code>separator</code> Custom segment identifier, currently only allows one delimiter to be set. Default is \n
+              - <code>max_tokens</code> Maximum length (token) defaults to 1000
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/document/create_by_text"
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/create_by_text' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "text","text": "text","indexing_technique": "high_quality","process_rule": {"mode": "automatic"}}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/create_by_text' \
+    --header 'Authorization: Bearer {api_key}' \
+    --header 'Content-Type: application/json' \
+    --data-raw '{
+        "name": "text",
+        "text": "text",
+        "indexing_technique": "high_quality",
+        "process_rule": {
+            "mode": "automatic"
+        }
+    }'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "document": {
+        "id": "",
+        "position": 1,
+        "data_source_type": "upload_file",
+        "data_source_info": {
+            "upload_file_id": ""
+        },
+        "dataset_process_rule_id": "",
+        "name": "text.txt",
+        "created_from": "api",
+        "created_by": "",
+        "created_at": 1695690280,
+        "tokens": 0,
+        "indexing_status": "waiting",
+        "error": null,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "archived": false,
+        "display_status": "queuing",
+        "word_count": 0,
+        "hit_count": 0,
+        "doc_form": "text_model"
+      },
+      "batch": ""
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/document/create_by_file'
+  method='POST'
+  title='Create documents from files'
+  name='#create_by_file'
+/>
+<Row>
+  <Col>
+    This api is based on an existing dataset and creates a new document through a file based on this dataset.
+
+    ### Params
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        Dataset ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='original_document_id' type='string' key='original_document_id'>
+        Source document ID (optional)
+          - Used to re-upload the document or modify the document cleaning and segmentation configuration. The missing information is copied from the source document
+          - The source document cannot be an archived document
+          - When original_document_id is passed in, the update operation is performed on behalf of the document. process_rule is a fillable item. If not filled in, the segmentation method of the source document will be used by defaul
+          - When original_document_id is not passed in, the new operation is performed on behalf of the document, and process_rule is required
+      </Property>
+      <Property name='file' type='multipart/form-data' key='file'>
+        Files that need to be uploaded.
+      </Property>
+      <Property name='indexing_technique' type='string' key='indexing_technique'>
+        Index mode
+          - <code>high_quality</code> High quality: embedding using embedding model, built as vector database index
+          - <code>economy</code> Economy: Build using inverted index of Keyword Table Index
+      </Property>
+      <Property name='process_rule' type='object' key='process_rule'>
+        Processing rules
+          - <code>mode</code> (string) Cleaning, segmentation mode, automatic / custom
+          - <code>rules</code> (object) Custom rules (in automatic mode, this field is empty)
+            - <code>pre_processing_rules</code> (array[object]) Preprocessing rules
+              - <code>id</code> (string) Unique identifier for the preprocessing rule
+                - enumerate 
+                  - <code>remove_extra_spaces</code> Replace consecutive spaces, newlines, tabs
+                  - <code>remove_urls_emails</code> Delete URL, email address
+              - <code>enabled</code> (bool) Whether to select this rule or not. If no document ID is passed in, it represents the default value.
+            - <code>segmentation</code> (object) segmentation rules
+              - <code>separator</code> Custom segment identifier, currently only allows one delimiter to be set. Default is \n
+              - <code>max_tokens</code> Maximum length (token) defaults to 1000
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/document/create_by_file"
+      targetCode={`curl --location POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/create_by_file' \\\n--header 'Authorization: Bearer {api_key}' \\\n--form 'data="{"name":"Dify","indexing_technique":"high_quality","process_rule":{"rules":{"pre_processing_rules":[{"id":"remove_extra_spaces","enabled":true},{"id":"remove_urls_emails","enabled":true}],"segmentation":{"separator":"###","max_tokens":500}},"mode":"custom"}}";type=text/plain' \\\n--form 'file=@"/path/to/file"'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/create_by_file' \
+    --header 'Authorization: Bearer {api_key}' \
+    --form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \
+    --form 'file=@"/path/to/file"'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "document": {
+        "id": "",
+        "position": 1,
+        "data_source_type": "upload_file",
+        "data_source_info": {
+          "upload_file_id": ""
+        },
+        "dataset_process_rule_id": "",
+        "name": "Dify.txt",
+        "created_from": "api",
+        "created_by": "",
+        "created_at": 1695308667,
+        "tokens": 0,
+        "indexing_status": "waiting",
+        "error": null,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "archived": false,
+        "display_status": "queuing",
+        "word_count": 0,
+        "hit_count": 0,
+        "doc_form": "text_model"
+      },
+      "batch": ""
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents/{document_id}/update_by_text'
+  method='POST'
+  title='Update document via text'
+  name='#update_by_text'
+/>
+<Row>
+  <Col>
+    This api is based on an existing dataset and updates the document through text based on this dataset.
+
+    ### Params
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        Dataset ID
+      </Property>
+      <Property name='document_id' type='string' key='document_id'>
+        Document ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='name' type='string' key='name'>
+        Document name (optional)
+      </Property>
+      <Property name='text' type='string' key='text'>
+        Document content (optional)
+      </Property>
+      <Property name='process_rule' type='object' key='process_rule'>
+        Processing rules
+          - <code>mode</code> (string) Cleaning, segmentation mode, automatic / custom
+          - <code>rules</code> (object) Custom rules (in automatic mode, this field is empty)
+            - <code>pre_processing_rules</code> (array[object]) Preprocessing rules
+              - <code>id</code> (string) Unique identifier for the preprocessing rule
+                - enumerate 
+                  - <code>remove_extra_spaces</code> Replace consecutive spaces, newlines, tabs
+                  - <code>remove_urls_emails</code> Delete URL, email address
+              - <code>enabled</code> (bool) Whether to select this rule or not. If no document ID is passed in, it represents the default value.
+            - <code>segmentation</code> (object) segmentation rules
+              - <code>separator</code> Custom segment identifier, currently only allows one delimiter to be set. Default is \n
+              - <code>max_tokens</code> Maximum length (token) defaults to 1000
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/documents/{document_id}/update_by_text"
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/update_by_text' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name","text": "text"}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}/update_by_text' \
+    --header 'Authorization: Bearer {api_key}' \
+    --header 'Content-Type: application/json' \
+    --data-raw '{
+        "name": "name",
+        "text": "text"
+    }'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "document": {
+        "id": "",
+        "position": 1,
+        "data_source_type": "upload_file",
+        "data_source_info": {
+          "upload_file_id": ""
+        },
+        "dataset_process_rule_id": "",
+        "name": "name.txt",
+        "created_from": "api",
+        "created_by": "",
+        "created_at": 1695308667,
+        "tokens": 0,
+        "indexing_status": "waiting",
+        "error": null,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "archived": false,
+        "display_status": "queuing",
+        "word_count": 0,
+        "hit_count": 0,
+        "doc_form": "text_model"
+      },
+      "batch": ""
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents/{document_id}/update_by_file'
+  method='POST'
+  title='Update a document from a file'
+  name='#update_by_file'
+/>
+<Row>
+  <Col>
+    This api is based on an existing dataset, and updates documents through files based on this dataset
+
+    ### Params
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        Dataset ID
+      </Property>
+      <Property name='document_id' type='string' key='document_id'>
+        Document ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='name' type='string' key='name'>
+        Document name (optional)
+      </Property>
+      <Property name='file' type='multipart/form-data' key='file'>
+        Files to be uploaded
+      </Property>
+      <Property name='process_rule' type='object' key='process_rule'>
+        Processing rules
+          - <code>mode</code> (string) Cleaning, segmentation mode, automatic / custom
+          - <code>rules</code> (object) Custom rules (in automatic mode, this field is empty)
+            - <code>pre_processing_rules</code> (array[object]) Preprocessing rules
+              - <code>id</code> (string) Unique identifier for the preprocessing rule
+                - enumerate 
+                  - <code>remove_extra_spaces</code> Replace consecutive spaces, newlines, tabs
+                  - <code>remove_urls_emails</code> Delete URL, email address
+              - <code>enabled</code> (bool) Whether to select this rule or not. If no document ID is passed in, it represents the default value.
+            - <code>segmentation</code> (object) segmentation rules
+              - <code>separator</code> Custom segment identifier, currently only allows one delimiter to be set. Default is \n
+              - <code>max_tokens</code> Maximum length (token) defaults to 1000
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/documents/{document_id}/update_by_file"
+      targetCode={`curl --location POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/{document_id}/create_by_file' \\\n--header 'Authorization: Bearer {api_key}' \\\n--form 'data="{"name":"Dify","indexing_technique":"high_quality","process_rule":{"rules":{"pre_processing_rules":[{"id":"remove_extra_spaces","enabled":true},{"id":"remove_urls_emails","enabled":true}],"segmentation":{"separator":"###","max_tokens":500}},"mode":"custom"}}";type=text/plain' \\\n--form 'file=@"/path/to/file"'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/{document_id}/create_by_file' \
+    --header 'Authorization: Bearer {api_key}' \
+    --form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \
+    --form 'file=@"/path/to/file"'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "document": {
+        "id": "",
+        "position": 1,
+        "data_source_type": "upload_file",
+        "data_source_info": {
+          "upload_file_id": ""
+        },
+        "dataset_process_rule_id": "",
+        "name": "Dify.txt",
+        "created_from": "api",
+        "created_by": "",
+        "created_at": 1695308667,
+        "tokens": 0,
+        "indexing_status": "waiting",
+        "error": null,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "archived": false,
+        "display_status": "queuing",
+        "word_count": 0,
+        "hit_count": 0,
+        "doc_form": "text_model"
+      },
+      "batch": "20230921150427533684"
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/batch/{batch}/indexing-status'
+  method='GET'
+  title='Get document embedding status (progress)'
+  name='#indexing_status'
+/>
+<Row>
+  <Col>
+    ### Params
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        Dataset ID
+      </Property>
+      <Property name='batch' type='string' key='batch'>
+        Batch number of uploaded documents
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="GET" 
+      label="/datasets/{dataset_id}/batch/{batch}/indexing-status"
+      targetCode={`curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{batch}/indexing-status' \\\n--header 'Authorization: Bearer {api_key}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request GET 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{batch}/indexing-status' \
+    --header 'Authorization: Bearer {api_key}' \
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "data":[{
+        "id": "",
+        "indexing_status": "indexing", 
+        "processing_started_at": 1681623462.0,
+        "parsing_completed_at": 1681623462.0,
+        "cleaning_completed_at": 1681623462.0,
+        "splitting_completed_at": 1681623462.0,
+        "completed_at": null,
+        "paused_at": null,
+        "error": null,
+        "stopped_at": null,
+        "completed_segments": 24,
+        "total_segments": 100
+      }]
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents/{document_id}'
+  method='DELETE'
+  title='Delete document'
+  name='#delete_document'
+/>
+<Row>
+  <Col>
+    ### Params
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        Dataset ID
+      </Property>
+      <Property name='document_id' type='string' key='document_id'>
+        Document ID
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="DELETE" 
+      label="/datasets/{dataset_id}/documents/{document_id}"
+      targetCode={`curl --location --request DELETE '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}' \\\n--header 'Authorization: Bearer {api_key}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request DELETE 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}' \
+    --header 'Authorization: Bearer {api_key}' \
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "result": "success"
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents'
+  method='GET'
+  title='Dataset document list'
+  name='#dataset_document_list'
+/>
+<Row>
+  <Col>
+    ### Params
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        Dataset ID
+      </Property>
+    </Properties>
+
+    ### Query
+    <Properties>
+      <Property name='keyword' type='string' key='keyword'>
+        Search keywords, currently only search document names(optional)
+      </Property>
+      <Property name='page' type='string' key='page'>
+        Page number(optional)
+      </Property>
+      <Property name='limit' type='string' key='limit'>
+        Number of items returned, default 20, range 1-100(optional)
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="GET" 
+      label="/datasets/{dataset_id}/documents"
+      targetCode={`curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents' \\\n--header 'Authorization: Bearer {api_key}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request GET 'https://api.dify.ai/v1/datasets/{dataset_id}/documents' \
+    --header 'Authorization: Bearer {api_key}' \
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "data": [
+        {
+          "id": "",
+          "position": 1,
+          "data_source_type": "file_upload",
+          "data_source_info": null,
+          "dataset_process_rule_id": null,
+          "name": "dify",
+          "created_from": "",
+          "created_by": "",
+          "created_at": 1681623639,
+          "tokens": 0,
+          "indexing_status": "waiting",
+          "error": null,
+          "enabled": true,
+          "disabled_at": null,
+          "disabled_by": null,
+          "archived": false
+        },
+      ],
+      "has_more": false,
+      "limit": 20,
+      "total": 9,
+      "page": 1
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents/{document_id}/segments'
+  method='POST'
+  title='Add segment'
+  name='#create_new_segment'
+/>
+<Row>
+  <Col>
+    ### Params
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        Dataset ID
+      </Property>
+      <Property name='document_id' type='string' key='document_id'>
+        Document ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='segments' type='object list' key='segments'>
+        - <code>content</code> (text) Text content/question content, required
+        - <code>answer</code> (text) Answer content, if the mode of the data set is qa mode, pass the value(optional)
+        - <code>keywords</code> (list) Keywords(optional)
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/documents/{document_id}/segments"
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/segments' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"segments": [{"content": "1","answer": "1","keywords": ["a"]}]}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}/segments' \
+    --header 'Authorization: Bearer {api_key}' \
+    --header 'Content-Type: application/json' \
+    --data-raw '{
+      "segments": [
+        {
+          "content": "1",
+          "answer": "1",
+          "keywords": ["a"]
+        }
+      ]
+    }'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "data": [{
+        "id": "",
+        "position": 1,
+        "document_id": "",
+        "content": "1",
+        "answer": "1",
+        "word_count": 25,
+        "tokens": 0,
+        "keywords": [
+          "a"
+        ],
+        "index_node_id": "",
+        "index_node_hash": "",
+        "hit_count": 0,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "status": "completed",
+        "created_by": "",
+        "created_at": 1695312007,
+        "indexing_at": 1695312007,
+        "completed_at": 1695312007,
+        "error": null,
+        "stopped_at": null
+      }],
+      "doc_form": "text_model"
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Row>
+  <Col>
+    ### Error message
+    <Properties>
+      <Property name='code' type='string' key='code'>
+        Error code
+      </Property>
+    </Properties>
+    <Properties>
+      <Property name='status' type='number' key='status'>
+        Error status
+      </Property>
+    </Properties>
+    <Properties>
+      <Property name='message' type='string' key='message'>
+        Error message
+      </Property>
+    </Properties>
+  </Col>
+  <Col>
+    <CodeGroup title="Example">
+    ```json {{ title: 'Response' }}
+      {
+        "code": "no_file_uploaded",
+        "message": "Please upload your file.",
+        "status": 400
+      }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+<table className="max-w-auto border-collapse border border-slate-400" style={{ maxWidth: 'none', width: 'auto' }}>
+  <thead style={{ background: '#f9fafc' }}>
+    <tr>
+      <th class="p-2 border border-slate-300">code</th>
+      <th class="p-2 border border-slate-300">status</th>
+      <th class="p-2 border border-slate-300">message</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td class="p-2 border border-slate-300">no_file_uploaded</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">Please upload your file.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">too_many_files</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">Only one file is allowed.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">file_too_large</td>
+      <td class="p-2 border border-slate-300">413</td>
+      <td class="p-2 border border-slate-300">File size exceeded.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">unsupported_file_type</td>
+      <td class="p-2 border border-slate-300">415</td>
+      <td class="p-2 border border-slate-300">File type not allowed.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">high_quality_dataset_only</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">Current operation only supports 'high-quality' datasets.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">dataset_not_initialized</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">The dataset is still being initialized or indexing. Please wait a moment.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">archived_document_immutable</td>
+      <td class="p-2 border border-slate-300">403</td>
+      <td class="p-2 border border-slate-300">The archived document is not editable.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">dataset_name_duplicate</td>
+      <td class="p-2 border border-slate-300">409</td>
+      <td class="p-2 border border-slate-300">The dataset name already exists. Please modify your dataset name.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">invalid_action</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">Invalid action.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">document_already_finished</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">The document has been processed. Please refresh the page or go to the document details.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">document_indexing</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">The document is being processed and cannot be edited.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">invalid_metadata</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">The metadata content is incorrect. Please check and verify.</td>
+    </tr>
+  </tbody>
+</table>
+<div class="pb-4" />

web/app/(commonLayout)/datasets/template/template.zh.mdx

@@ -0,0 +1,883 @@
+import { CodeGroup } from '@/app/components/develop/code.tsx'
+import { Row, Col, Properties, Property, Heading, SubProperty, Paragraph } from '@/app/components/develop/md.tsx'
+
+# 数据集 API
+<br/>
+<br/>
+<Heading
+  url='/datasets'
+  method='POST'
+  title='创建空数据集'
+  name='#create_empty_dataset'
+/>
+<Row>
+  <Col>
+    ### Request Body
+    <Properties>
+      <Property name='name' type='string' key='name'>
+        数据集名称
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets"
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name"}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request POST 'https://api.dify.ai/v1/datasets' \
+    --header 'Authorization: Bearer {api_key}' \
+    --header 'Content-Type: application/json' \
+    --data-raw '{
+      "name": "name"
+    }'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "id": "",
+      "name": "name",
+      "description": null,
+      "provider": "vendor",
+      "permission": "only_me",
+      "data_source_type": null,
+      "indexing_technique": null,
+      "app_count": 0,
+      "document_count": 0,
+      "word_count": 0,
+      "created_by": "",
+      "created_at": 1695636173,
+      "updated_by": "",
+      "updated_at": 1695636173,
+      "embedding_model": null,
+      "embedding_model_provider": null,
+      "embedding_available": null
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets'
+  method='GET'
+  title='数据集列表'
+  name='#dataset_list'
+/>
+<Row>
+  <Col>
+    ### Query
+    <Properties>
+      <Property name='page' type='string' key='page'>
+        页码
+      </Property>
+      <Property name='limit' type='string' key='limit'>
+        返回条数，默认 20，范围 1-100
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets"
+      targetCode={`curl --location --request GET '${props.apiBaseUrl}/datasets?page=1&limit=20' \\\n--header 'Authorization: Bearer {api_key}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request GET 'https://api.dify.ai/v1/datasets?page=1&limit=20' \
+    --header 'Authorization: Bearer {api_key}'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "data": [
+        {
+          "id": "",
+          "name": "数据集名称",
+          "description": "描述信息",
+          "permission": "only_me",
+          "data_source_type": "upload_file",
+          "indexing_technique": "",
+          "app_count": 2,
+          "document_count": 10,
+          "word_count": 1200,
+          "created_by": "",
+          "created_at": "",
+          "updated_by": "",
+          "updated_at": ""
+        },
+        ...
+      ],
+      "has_more": true,
+      "limit": 20,
+      "total": 50,
+      "page": 1
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/document/create_by_text'
+  method='POST'
+  title='通过文本创建文档'
+  name='#create_by_text'
+/>
+<Row>
+  <Col>
+    此接口基于已存在数据集，在此数据集的基础上通过文本创建新的文档
+
+    ### Path
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        数据集 ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='name' type='string' key='name'>
+        文档名称
+      </Property>
+      <Property name='text' type='string' key='text'>
+        文档内容
+      </Property>
+      <Property name='indexing_technique' type='string' key='indexing_technique'>
+        索引方式
+          - <code>high_quality</code> 高质量：使用  embedding 模型进行嵌入，构建为向量数据库索引
+          - <code>economy</code> 经济：使用 Keyword Table Index 的倒排索引进行构建
+      </Property>
+      <Property name='process_rule' type='object' key='process_rule'>
+        处理规则
+          - <code>mode</code> (string) 清洗、分段模式 ，automatic 自动 / custom 自定义
+          - <code>rules</code> (object) 自定义规则（自动模式下，该字段为空）
+            - <code>pre_processing_rules</code> (array[object]) 预处理规则
+              - <code>id</code> (string) 预处理规则的唯一标识符
+                - 枚举： 
+                  - <code>remove_extra_spaces</code> 替换连续空格、换行符、制表符
+                  - <code>remove_urls_emails</code> 删除 URL、电子邮件地址
+              - <code>enabled</code> (bool) 是否选中该规则，不传入文档 ID 时代表默认值
+            - <code>segmentation</code> (object) 分段规则
+              - <code>separator</code> 自定义分段标识符，目前仅允许设置一个分隔符。默认为 \n
+              - <code>max_tokens</code> 最大长度 (token) 默认为 1000
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/document/create_by_text"
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/create_by_text' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "text","text": "text","indexing_technique": "high_quality","process_rule": {"mode": "automatic"}}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/create_by_text' \
+    --header 'Authorization: Bearer {api_key}' \
+    --header 'Content-Type: application/json' \
+    --data-raw '{
+        "name": "text",
+        "text": "text",
+        "indexing_technique": "high_quality",
+        "process_rule": {
+            "mode": "automatic"
+        }
+    }'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "document": {
+        "id": "",
+        "position": 1,
+        "data_source_type": "upload_file",
+        "data_source_info": {
+            "upload_file_id": ""
+        },
+        "dataset_process_rule_id": "",
+        "name": "text.txt",
+        "created_from": "api",
+        "created_by": "",
+        "created_at": 1695690280,
+        "tokens": 0,
+        "indexing_status": "waiting",
+        "error": null,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "archived": false,
+        "display_status": "queuing",
+        "word_count": 0,
+        "hit_count": 0,
+        "doc_form": "text_model"
+      },
+      "batch": ""
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/document/create_by_file'
+  method='POST'
+  title='通过文件创建文档 '
+  name='#create_by_file'
+/>
+<Row>
+  <Col>
+    此接口基于已存在数据集，在此数据集的基础上通过文件创建新的文档
+
+    ### Path
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        数据集 ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='original_document_id' type='string' key='original_document_id'>
+        源文档 ID （选填）
+          - 用于重新上传文档或修改文档清洗、分段配置，缺失的信息从源文档复制
+          - 源文档不可为归档的文档
+          - 当传入 <code>original_document_id</code> 时，代表文档进行更新操作，<code>process_rule</code> 为可填项目，不填默认使用源文档的分段方式
+          - 未传入 <code>original_document_id</code> 时，代表文档进行新增操作，<code>process_rule</code> 为必填
+      </Property>
+      <Property name='file' type='multipart/form-data' key='file'>
+        需要上传的文件。
+      </Property>
+      <Property name='indexing_technique' type='string' key='indexing_technique'>
+        索引方式
+          - <code>high_quality</code> 高质量：使用  embedding 模型进行嵌入，构建为向量数据库索引
+          - <code>economy</code> 经济：使用 Keyword Table Index 的倒排索引进行构建
+      </Property>
+      <Property name='process_rule' type='object' key='process_rule'>
+        处理规则
+          - <code>mode</code> (string) 清洗、分段模式 ，automatic 自动 / custom 自定义
+          - <code>rules</code> (object) 自定义规则（自动模式下，该字段为空）
+            - <code>pre_processing_rules</code> (array[object]) 预处理规则
+              - <code>id</code> (string) 预处理规则的唯一标识符
+                - 枚举： 
+                  - <code>remove_extra_spaces</code> 替换连续空格、换行符、制表符
+                  - <code>remove_urls_emails</code> 删除 URL、电子邮件地址
+              - <code>enabled</code> (bool) 是否选中该规则，不传入文档 ID 时代表默认值
+            - <code>segmentation</code> (object) 分段规则
+              - <code>separator</code> 自定义分段标识符，目前仅允许设置一个分隔符。默认为 \n
+              - <code>max_tokens</code> 最大长度 (token) 默认为 1000
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/document/create_by_file"
+      targetCode={`curl --location POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/create_by_file' \\\n--header 'Authorization: Bearer {api_key}' \\\n--form 'data="{"name":"Dify","indexing_technique":"high_quality","process_rule":{"rules":{"pre_processing_rules":[{"id":"remove_extra_spaces","enabled":true},{"id":"remove_urls_emails","enabled":true}],"segmentation":{"separator":"###","max_tokens":500}},"mode":"custom"}}";type=text/plain' \\\n--form 'file=@"/path/to/file"'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/create_by_file' \
+    --header 'Authorization: Bearer {api_key}' \
+    --form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \
+    --form 'file=@"/path/to/file"'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "document": {
+        "id": "",
+        "position": 1,
+        "data_source_type": "upload_file",
+        "data_source_info": {
+          "upload_file_id": ""
+        },
+        "dataset_process_rule_id": "",
+        "name": "Dify.txt",
+        "created_from": "api",
+        "created_by": "",
+        "created_at": 1695308667,
+        "tokens": 0,
+        "indexing_status": "waiting",
+        "error": null,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "archived": false,
+        "display_status": "queuing",
+        "word_count": 0,
+        "hit_count": 0,
+        "doc_form": "text_model"
+      },
+      "batch": ""
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents/{document_id}/update_by_text'
+  method='POST'
+  title='通过文本更新文档 '
+  name='#update_by_text'
+/>
+<Row>
+  <Col>
+    此接口基于已存在数据集，在此数据集的基础上通过文本更新文档
+
+    ### Path
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        数据集 ID
+      </Property>
+      <Property name='document_id' type='string' key='document_id'>
+        文档 ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='name' type='string' key='name'>
+        文档名称 （选填）
+      </Property>
+      <Property name='text' type='string' key='text'>
+        文档内容（选填）
+      </Property>
+      <Property name='process_rule' type='object' key='process_rule'>
+        处理规则（选填）
+          - <code>mode</code> (string) 清洗、分段模式 ，automatic 自动 / custom 自定义
+          - <code>rules</code> (object) 自定义规则（自动模式下，该字段为空）
+            - <code>pre_processing_rules</code> (array[object]) 预处理规则
+              - <code>id</code> (string) 预处理规则的唯一标识符
+                - 枚举： 
+                  - <code>remove_extra_spaces</code> 替换连续空格、换行符、制表符
+                  - <code>remove_urls_emails</code> 删除 URL、电子邮件地址
+              - <code>enabled</code> (bool) 是否选中该规则，不传入文档 ID 时代表默认值
+            - <code>segmentation</code> (object) 分段规则
+              - <code>separator</code> 自定义分段标识符，目前仅允许设置一个分隔符。默认为 \n
+              - <code>max_tokens</code> 最大长度 (token) 默认为 1000
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/documents/{document_id}/update_by_text"
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/update_by_text' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"name": "name","text": "text"}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}/update_by_text' \
+    --header 'Authorization: Bearer {api_key}' \
+    --header 'Content-Type: application/json' \
+    --data-raw '{
+        "name": "name",
+        "text": "text"
+    }'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "document": {
+        "id": "",
+        "position": 1,
+        "data_source_type": "upload_file",
+        "data_source_info": {
+          "upload_file_id": ""
+        },
+        "dataset_process_rule_id": "",
+        "name": "name.txt",
+        "created_from": "api",
+        "created_by": "",
+        "created_at": 1695308667,
+        "tokens": 0,
+        "indexing_status": "waiting",
+        "error": null,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "archived": false,
+        "display_status": "queuing",
+        "word_count": 0,
+        "hit_count": 0,
+        "doc_form": "text_model"
+      },
+      "batch": ""
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents/{document_id}/update_by_file'
+  method='POST'
+  title='通过文件更新文档  '
+  name='#update_by_file'
+/>
+<Row>
+  <Col>
+    此接口基于已存在数据集，在此数据集的基础上通过文件更新文档的操作。
+
+    ### Path
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        数据集 ID
+      </Property>
+      <Property name='document_id' type='string' key='document_id'>
+        文档 ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='name' type='string' key='name'>
+        文档名称 （选填）
+      </Property>
+      <Property name='file' type='multipart/form-data' key='file'>
+        需要上传的文件
+      </Property>
+      <Property name='process_rule' type='object' key='process_rule'>
+        处理规则（选填）
+          - <code>mode</code> (string) 清洗、分段模式 ，automatic 自动 / custom 自定义
+          - <code>rules</code> (object) 自定义规则（自动模式下，该字段为空）
+            - <code>pre_processing_rules</code> (array[object]) 预处理规则
+              - <code>id</code> (string) 预处理规则的唯一标识符
+                - 枚举： 
+                  - <code>remove_extra_spaces</code> 替换连续空格、换行符、制表符
+                  - <code>remove_urls_emails</code> 删除 URL、电子邮件地址
+              - <code>enabled</code> (bool) 是否选中该规则，不传入文档 ID 时代表默认值
+            - <code>segmentation</code> (object) 分段规则
+              - <code>separator</code> 自定义分段标识符，目前仅允许设置一个分隔符。默认为 \n
+              - <code>max_tokens</code> 最大长度 (token) 默认为 1000
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/documents/{document_id}/update_by_file"
+      targetCode={`curl --location POST '${props.apiBaseUrl}/datasets/{dataset_id}/document/{document_id}/create_by_file' \\\n--header 'Authorization: Bearer {api_key}' \\\n--form 'data="{"name":"Dify","indexing_technique":"high_quality","process_rule":{"rules":{"pre_processing_rules":[{"id":"remove_extra_spaces","enabled":true},{"id":"remove_urls_emails","enabled":true}],"segmentation":{"separator":"###","max_tokens":500}},"mode":"custom"}}";type=text/plain' \\\n--form 'file=@"/path/to/file"'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location POST 'https://api.dify.ai/v1/datasets/{dataset_id}/document/{document_id}/create_by_file' \
+    --header 'Authorization: Bearer {api_key}' \
+    --form 'data="{\"name\":\"Dify\",\"indexing_technique\":\"high_quality\",\"process_rule\":{\"rules\":{\"pre_processing_rules\":[{\"id\":\"remove_extra_spaces\",\"enabled\":true},{\"id\":\"remove_urls_emails\",\"enabled\":true}],\"segmentation\":{\"separator\":\"###\",\"max_tokens\":500}},\"mode\":\"custom\"}}";type=text/plain' \
+    --form 'file=@"/path/to/file"'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "document": {
+        "id": "",
+        "position": 1,
+        "data_source_type": "upload_file",
+        "data_source_info": {
+          "upload_file_id": ""
+        },
+        "dataset_process_rule_id": "",
+        "name": "Dify.txt",
+        "created_from": "api",
+        "created_by": "",
+        "created_at": 1695308667,
+        "tokens": 0,
+        "indexing_status": "waiting",
+        "error": null,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "archived": false,
+        "display_status": "queuing",
+        "word_count": 0,
+        "hit_count": 0,
+        "doc_form": "text_model"
+      },
+      "batch": "20230921150427533684"
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/batch/{batch}/indexing-status'
+  method='GET'
+  title='获取文档嵌入状态（进度）'
+  name='#indexing_status'
+/>
+<Row>
+  <Col>
+    ### Path
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        数据集 ID
+      </Property>
+      <Property name='batch' type='string' key='batch'>
+        上传文档的批次号
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="GET" 
+      label="/datasets/{dataset_id}/batch/{batch}/indexing-status"
+      targetCode={`curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{batch}/indexing-status' \\\n--header 'Authorization: Bearer {api_key}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request GET 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{batch}/indexing-status' \
+    --header 'Authorization: Bearer {api_key}' \
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "data":[{
+        "id": "",
+        "indexing_status": "indexing", 
+        "processing_started_at": 1681623462.0,
+        "parsing_completed_at": 1681623462.0,
+        "cleaning_completed_at": 1681623462.0,
+        "splitting_completed_at": 1681623462.0,
+        "completed_at": null,
+        "paused_at": null,
+        "error": null,
+        "stopped_at": null,
+        "completed_segments": 24,
+        "total_segments": 100
+      }]
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents/{document_id}'
+  method='DELETE'
+  title='删除文档'
+  name='#delete_document'
+/>
+<Row>
+  <Col>
+    ### Path
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        数据集 ID
+      </Property>
+      <Property name='document_id' type='string' key='document_id'>
+        文档 ID
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="DELETE" 
+      label="/datasets/{dataset_id}/documents/{document_id}"
+      targetCode={`curl --location --request DELETE '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}' \\\n--header 'Authorization: Bearer {api_key}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request DELETE 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}' \
+    --header 'Authorization: Bearer {api_key}' \
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "result": "success"
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents'
+  method='GET'
+  title='数据集文档列表'
+  name='#dataset_document_list'
+/>
+<Row>
+  <Col>
+    ### Path
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        数据集 ID
+      </Property>
+    </Properties>
+
+    ### Query
+    <Properties>
+      <Property name='keyword' type='string' key='keyword'>
+        搜索关键词，可选，目前仅搜索文档名称
+      </Property>
+      <Property name='page' type='string' key='page'>
+        页码，可选
+      </Property>
+      <Property name='limit' type='string' key='limit'>
+        返回条数，可选，默认 20，范围 1-100
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="GET" 
+      label="/datasets/{dataset_id}/documents"
+      targetCode={`curl --location --request GET '${props.apiBaseUrl}/datasets/{dataset_id}/documents' \\\n--header 'Authorization: Bearer {api_key}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request GET 'https://api.dify.ai/v1/datasets/{dataset_id}/documents' \
+    --header 'Authorization: Bearer {api_key}' \
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "data": [
+        {
+          "id": "",
+          "position": 1,
+          "data_source_type": "file_upload",
+          "data_source_info": null,
+          "dataset_process_rule_id": null,
+          "name": "dify",
+          "created_from": "",
+          "created_by": "",
+          "created_at": 1681623639,
+          "tokens": 0,
+          "indexing_status": "waiting",
+          "error": null,
+          "enabled": true,
+          "disabled_at": null,
+          "disabled_by": null,
+          "archived": false
+        },
+      ],
+      "has_more": false,
+      "limit": 20,
+      "total": 9,
+      "page": 1
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Heading
+  url='/datasets/{dataset_id}/documents/{document_id}/segments'
+  method='POST'
+  title='新增分段'
+  name='#create_new_segment'
+/>
+<Row>
+  <Col>
+    ### Path
+    <Properties>
+      <Property name='dataset_id' type='string' key='dataset_id'>
+        数据集 ID
+      </Property>
+      <Property name='document_id' type='string' key='document_id'>
+        文档 ID
+      </Property>
+    </Properties>
+
+    ### Request Body
+    <Properties>
+      <Property name='segments' type='object list' key='segments'>
+        - <code>content</code> (text) 文本内容/问题内容，必填
+        - <code>answer</code> (text) 答案内容，非必填，如果数据集的模式为qa模式则传值
+        - <code>keywords</code> (list) 关键字，非必填
+      </Property>
+    </Properties>
+  </Col>
+  <Col sticky>
+    <CodeGroup 
+      title="Request" 
+      tag="POST" 
+      label="/datasets/{dataset_id}/documents/{document_id}/segments"
+      targetCode={`curl --location --request POST '${props.apiBaseUrl}/datasets/{dataset_id}/documents/{document_id}/segments' \\\n--header 'Authorization: Bearer {api_key}' \\\n--header 'Content-Type: application/json' \\\n--data-raw '{"segments": [{"content": "1","answer": "1","keywords": ["a"]}]}'`}
+    >
+    ```bash {{ title: 'cURL' }}
+    curl --location --request POST 'https://api.dify.ai/v1/datasets/{dataset_id}/documents/{document_id}/segments' \
+    --header 'Authorization: Bearer {api_key}' \
+    --header 'Content-Type: application/json' \
+    --data-raw '{
+      "segments": [
+        {
+          "content": "1",
+          "answer": "1",
+          "keywords": ["a"]
+        }
+      ]
+    }'
+    ```
+    </CodeGroup>
+    <CodeGroup title="Response">
+    ```json {{ title: 'Response' }}
+    {
+      "data": [{
+        "id": "",
+        "position": 1,
+        "document_id": "",
+        "content": "1",
+        "answer": "1",
+        "word_count": 25,
+        "tokens": 0,
+        "keywords": [
+            "a"
+        ],
+        "index_node_id": "",
+        "index_node_hash": "",
+        "hit_count": 0,
+        "enabled": true,
+        "disabled_at": null,
+        "disabled_by": null,
+        "status": "completed",
+        "created_by": "",
+        "created_at": 1695312007,
+        "indexing_at": 1695312007,
+        "completed_at": 1695312007,
+        "error": null,
+        "stopped_at": null
+      }],
+      "doc_form": "text_model"
+    }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+
+---
+
+<Row>
+  <Col>
+    ### 错误信息
+    <Properties>
+      <Property name='code' type='string' key='code'>
+        返回的错误代码
+      </Property>
+    </Properties>
+    <Properties>
+      <Property name='status' type='number' key='status'>
+        返回的错误状态
+      </Property>
+    </Properties>
+    <Properties>
+      <Property name='message' type='string' key='message'>
+        返回的错误信息
+      </Property>
+    </Properties>
+  </Col>
+  <Col>
+    <CodeGroup title="Example">
+    ```json {{ title: 'Response' }}
+      {
+        "code": "no_file_uploaded",
+        "message": "Please upload your file.",
+        "status": 400
+      }
+    ```
+    </CodeGroup>
+  </Col>
+</Row>
+<table className="max-w-auto border-collapse border border-slate-400" style={{ maxWidth: 'none', width: 'auto' }}>
+  <thead style={{ background: '#f9fafc' }}>
+    <tr>
+      <th class="p-2 border border-slate-300">code</th>
+      <th class="p-2 border border-slate-300">status</th>
+      <th class="p-2 border border-slate-300">message</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td class="p-2 border border-slate-300">no_file_uploaded</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">Please upload your file.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">too_many_files</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">Only one file is allowed.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">file_too_large</td>
+      <td class="p-2 border border-slate-300">413</td>
+      <td class="p-2 border border-slate-300">File size exceeded.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">unsupported_file_type</td>
+      <td class="p-2 border border-slate-300">415</td>
+      <td class="p-2 border border-slate-300">File type not allowed.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">high_quality_dataset_only</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">Current operation only supports 'high-quality' datasets.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">dataset_not_initialized</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">The dataset is still being initialized or indexing. Please wait a moment.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">archived_document_immutable</td>
+      <td class="p-2 border border-slate-300">403</td>
+      <td class="p-2 border border-slate-300">The archived document is not editable.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">dataset_name_duplicate</td>
+      <td class="p-2 border border-slate-300">409</td>
+      <td class="p-2 border border-slate-300">The dataset name already exists. Please modify your dataset name.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">invalid_action</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">Invalid action.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">document_already_finished</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">The document has been processed. Please refresh the page or go to the document details.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">document_indexing</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">The document is being processed and cannot be edited.</td>
+    </tr>
+    <tr>
+      <td class="p-2 border border-slate-300">invalid_metadata</td>
+      <td class="p-2 border border-slate-300">400</td>
+      <td class="p-2 border border-slate-300">The metadata content is incorrect. Please check and verify.</td>
+    </tr>
+  </tbody>
+</table>
+<div class="pb-4" />

web/app/components/app/chat/copy-btn/index.tsx

@@ -1,9 +1,10 @@
 'use client'
-import React from 'react'
+import { useRef, useState } from 'react'
 import { t } from 'i18next'
 import copy from 'copy-to-clipboard'
 import s from './style.module.css'
 import Tooltip from '@/app/components/base/tooltip'
+import { randomString } from '@/utils'
 
 type ICopyBtnProps = {
   value: string
@@ -16,12 +17,13 @@ const CopyBtn = ({
   className,
   isPlain,
 }: ICopyBtnProps) => {
-  const [isCopied, setIsCopied] = React.useState(false)
+  const [isCopied, setIsCopied] = useState(false)
+  const selector = useRef(`copy-tooltip-${randomString(4)}`)
 
   return (
     <div className={`${className}`}>
       <Tooltip
-        selector={`copy-btn-tooltip-${value}`}
+        selector={selector.current}
         content={(isCopied ? t('appApi.copied') : t('appApi.copy')) as string}
         className='z-10'
       >

web/app/components/app/configuration/base/icons/var-icon.tsx

@@ -1,11 +0,0 @@
-'use client'
-import React, { FC } from 'react'
-
-const VarIcon: FC = () => {
-  return (
-    <svg width="16" height="17" viewBox="0 0 16 17" fill="none" xmlns="http://www.w3.org/2000/svg">
-      <path fillRule="evenodd" clipRule="evenodd" d="M3.71922 2.96714C3.91321 3.05239 4.06546 3.21112 4.14255 3.4085C4.21964 3.60588 4.21528 3.82578 4.13042 4.01994C3.51419 5.43299 3.1974 6.95838 3.20002 8.49994C3.20002 10.0943 3.53282 11.6087 4.13122 12.9799C4.20853 13.173 4.20766 13.3885 4.1288 13.5809C4.04993 13.7732 3.89927 13.9274 3.70873 14.0106C3.51819 14.0938 3.30274 14.0995 3.10802 14.0266C2.91331 13.9537 2.75464 13.8078 2.66562 13.6199C1.96073 12.0052 1.59791 10.2619 1.60002 8.49994C1.60002 6.68074 1.98002 4.94794 2.66562 3.37994C2.70767 3.28364 2.76829 3.19658 2.84401 3.12371C2.91973 3.05085 3.00906 2.99361 3.1069 2.95528C3.20474 2.91694 3.30917 2.89826 3.41424 2.9003C3.5193 2.90233 3.62293 2.92505 3.71922 2.96714ZM10.368 6.09994C10.0082 6.10002 9.65293 6.18102 9.32862 6.33695C9.0043 6.49288 8.7192 6.71974 8.49441 7.00074L8.23201 7.32874L8.14321 7.10554C8.02447 6.80879 7.81958 6.55441 7.55494 6.37518C7.2903 6.19594 6.97804 6.10008 6.65841 6.09994H6.40001C6.18784 6.09994 5.98436 6.18423 5.83433 6.33426C5.6843 6.48429 5.60002 6.68777 5.60002 6.89994C5.60002 7.11212 5.6843 7.3156 5.83433 7.46563C5.98436 7.61566 6.18784 7.69994 6.40001 7.69994H6.65841L7.08401 8.76394L6.25602 9.79994C6.18103 9.8936 6.08594 9.96919 5.97779 10.0211C5.86964 10.073 5.75119 10.1 5.63122 10.0999H5.60002C5.38784 10.0999 5.18436 10.1842 5.03433 10.3343C4.8843 10.4843 4.80001 10.6878 4.80001 10.8999C4.80001 11.1121 4.8843 11.3156 5.03433 11.4656C5.18436 11.6157 5.38784 11.6999 5.60002 11.6999H5.63122C5.99107 11.6999 6.34629 11.6189 6.67061 11.4629C6.99493 11.307 7.28003 11.0801 7.50481 10.7991L7.76721 10.4711L7.85601 10.6943C7.97481 10.9912 8.17982 11.2457 8.44462 11.4249C8.70942 11.6042 9.02185 11.7 9.34161 11.6999H9.60001C9.81219 11.6999 10.0157 11.6157 10.1657 11.4656C10.3157 11.3156 10.4 11.1121 10.4 10.8999C10.4 10.6878 10.3157 10.4843 10.1657 10.3343C10.0157 10.1842 9.81219 10.0999 9.60001 10.0999H9.34161L8.91602 9.03594L9.74401 7.99994C9.819 7.90629 9.91409 7.8307 10.0222 7.77877C10.1304 7.72684 10.2488 7.6999 10.3688 7.69994H10.4C10.6122 7.69994 10.8157 7.61566 10.9657 7.46563C11.1157 7.3156 11.2 7.11212 11.2 6.89994C11.2 6.68777 11.1157 6.48429 10.9657 6.33426C10.8157 6.18423 10.6122 6.09994 10.4 6.09994H10.3688H10.368ZM11.8672 4.01994C11.7837 3.82571 11.7804 3.6063 11.8581 3.40966C11.9359 3.21303 12.0883 3.05517 12.2821 2.97059C12.4759 2.88602 12.6953 2.8816 12.8923 2.9583C13.0894 3.03501 13.248 3.1866 13.3336 3.37994C14.0388 4.99468 14.4019 6.73795 14.4 8.49994C14.4 10.3191 14.02 12.0519 13.3344 13.6199C13.2946 13.7194 13.2352 13.8098 13.1597 13.8859C13.0843 13.9619 12.9943 14.0221 12.8952 14.0627C12.7961 14.1033 12.6898 14.1236 12.5827 14.1224C12.4756 14.1212 12.3698 14.0985 12.2716 14.0556C12.1735 14.0127 12.0849 13.9506 12.0112 13.8728C11.9375 13.7951 11.8802 13.7033 11.8426 13.603C11.805 13.5027 11.788 13.3959 11.7925 13.2888C11.797 13.1818 11.8229 13.0768 11.8688 12.9799C12.4853 11.567 12.8024 10.0416 12.8 8.49994C12.8 6.90554 12.4672 5.39114 11.868 4.01994H11.8672Z" fill="#FF8A4C" />
-    </svg>
-  )
-}
-export default React.memo(VarIcon)

web/app/components/app/configuration/base/warning-mask/cannot-query-dataset.tsx

@@ -0,0 +1,31 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import WarningMask from '.'
+import Button from '@/app/components/base/button'
+
+export type IFormattingChangedProps = {
+  onConfirm: () => void
+}
+
+const FormattingChanged: FC<IFormattingChangedProps> = ({
+  onConfirm,
+}) => {
+  const { t } = useTranslation()
+
+  return (
+    <WarningMask
+      title={t('appDebug.feature.dataSet.queryVariable.unableToQueryDataSet')}
+      description={t('appDebug.feature.dataSet.queryVariable.unableToQueryDataSetTip')}
+      footer={
+        <div className='flex space-x-2'>
+          <Button type='primary' className='flex items-center justify-start !h-8 !w-[96px]' onClick={onConfirm}>
+            <span className='text-[13px] font-medium'>{t('appDebug.feature.dataSet.queryVariable.ok')}</span>
+          </Button>
+        </div>
+      }
+    />
+  )
+}
+export default React.memo(FormattingChanged)

web/app/components/app/configuration/base/warning-mask/formatting-changed.tsx

@@ -1,10 +1,11 @@
 'use client'
-import React, { FC } from 'react'
+import type { FC } from 'react'
+import React from 'react'
 import { useTranslation } from 'react-i18next'
 import WarningMask from '.'
 import Button from '@/app/components/base/button'
 
-export interface IFormattingChangedProps {
+export type IFormattingChangedProps = {
   onConfirm: () => void
   onCancel: () => void
 }
@@ -17,7 +18,7 @@ const icon = (
 
 const FormattingChanged: FC<IFormattingChangedProps> = ({
   onConfirm,
-  onCancel
+  onCancel,
 }) => {
   const { t } = useTranslation()
 

web/app/components/app/configuration/config-var/index.tsx

@@ -5,19 +5,24 @@ import { useTranslation } from 'react-i18next'
 import { Cog8ToothIcon, TrashIcon } from '@heroicons/react/24/outline'
 import { useBoolean } from 'ahooks'
 import type { Timeout } from 'ahooks/lib/useRequest/src/types'
+import { useContext } from 'use-context-selector'
 import Panel from '../base/feature-panel'
 import OperationBtn from '../base/operation-btn'
-import VarIcon from '../base/icons/var-icon'
 import EditModal from './config-modal'
 import IconTypeIcon from './input-type-icon'
 import type { IInputTypeIconProps } from './input-type-icon'
 import s from './style.module.css'
+import { BracketsX as VarIcon } from '@/app/components/base/icons/src/vender/line/development'
 import Tooltip from '@/app/components/base/tooltip'
 import type { PromptVariable } from '@/models/debug'
 import { DEFAULT_VALUE_MAX_LEN, getMaxVarNameLength } from '@/config'
 import { checkKeys, getNewVar } from '@/utils/var'
 import Switch from '@/app/components/base/switch'
 import Toast from '@/app/components/base/toast'
+import { HelpCircle } from '@/app/components/base/icons/src/vender/line/general'
+import ConfirmModal from '@/app/components/base/confirm/common'
+import ConfigContext from '@/context/debug-configuration'
+import { AppType } from '@/types/app'
 
 export type IConfigVarProps = {
   promptVariables: PromptVariable[]
@@ -29,6 +34,11 @@ let conflictTimer: Timeout
 
 const ConfigVar: FC<IConfigVarProps> = ({ promptVariables, readonly, onPromptVariablesChange }) => {
   const { t } = useTranslation()
+  const {
+    mode,
+    dataSets,
+  } = useContext(ConfigContext)
+
   const hasVar = promptVariables.length > 0
   const promptVariableObj = (() => {
     const obj: Record<string, boolean> = {}
@@ -127,10 +137,23 @@ const ConfigVar: FC<IConfigVarProps> = ({ promptVariables, readonly, onPromptVar
     onPromptVariablesChange?.([...promptVariables, newVar])
   }
 
-  const handleRemoveVar = (index: number) => {
+  const [isShowDeleteContextVarModal, { setTrue: showDeleteContextVarModal, setFalse: hideDeleteContextVarModal }] = useBoolean(false)
+  const [removeIndex, setRemoveIndex] = useState<number | null>(null)
+  const didRemoveVar = (index: number) => {
     onPromptVariablesChange?.(promptVariables.filter((_, i) => i !== index))
   }
 
+  const handleRemoveVar = (index: number) => {
+    const removeVar = promptVariables[index]
+
+    if (mode === AppType.completion && dataSets.length > 0 && removeVar.is_context_var) {
+      showDeleteContextVarModal()
+      setRemoveIndex(index)
+      return
+    }
+    didRemoveVar(index)
+  }
+
   const [currKey, setCurrKey] = useState<string | null>(null)
   const currItem = currKey ? promptVariables.find(item => item.key === currKey) : null
   const [isShowEditModal, { setTrue: showEditModal, setFalse: hideEditModal }] = useBoolean(false)
@@ -143,18 +166,17 @@ const ConfigVar: FC<IConfigVarProps> = ({ promptVariables, readonly, onPromptVar
     <Panel
       className="mt-4"
       headerIcon={
-        <VarIcon />
+        <VarIcon className='w-4 h-4 text-primary-500'/>
       }
       title={
-        <div className='flex items-center gap-2'>
-          <div>{t('appDebug.variableTitle')}</div>
+        <div className='flex items-center'>
+          <div className='ml-1 mr-1'>{t('appDebug.variableTitle')}</div>
           {!readonly && (
             <Tooltip htmlContent={<div className='w-[180px]'>
               {t('appDebug.variableTip')}
             </div>} selector='config-var-tooltip'>
-              <svg width="16" height="17" viewBox="0 0 16 17" fill="none" xmlns="http://www.w3.org/2000/svg">
-                <path d="M8.66667 11.1667H8V8.5H7.33333M8 5.83333H8.00667M14 8.5C14 9.28793 13.8448 10.0681 13.5433 10.7961C13.2417 11.5241 12.7998 12.1855 12.2426 12.7426C11.6855 13.2998 11.0241 13.7417 10.2961 14.0433C9.56815 14.3448 8.78793 14.5 8 14.5C7.21207 14.5 6.43185 14.3448 5.7039 14.0433C4.97595 13.7417 4.31451 13.2998 3.75736 12.7426C3.20021 12.1855 2.75825 11.5241 2.45672 10.7961C2.15519 10.0681 2 9.28793 2 8.5C2 6.9087 2.63214 5.38258 3.75736 4.25736C4.88258 3.13214 6.4087 2.5 8 2.5C9.5913 2.5 11.1174 3.13214 12.2426 4.25736C13.3679 5.38258 14 6.9087 14 8.5Z" stroke="#9CA3AF" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
-              </svg>
+              <HelpCircle className='w-3.5 h-3.5 text-gray-400'/>
+
             </Tooltip>
           )}
         </div>
@@ -185,7 +207,7 @@ const ConfigVar: FC<IConfigVarProps> = ({ promptVariables, readonly, onPromptVar
                 <tr key={index} className="h-9 leading-9">
                   <td className="w-[160px] border-b border-gray-100 pl-3">
                     <div className='flex items-center space-x-1'>
-                      <IconTypeIcon type={type as IInputTypeIconProps['type']} />
+                      <IconTypeIcon type={type as IInputTypeIconProps['type']} className='text-gray-400' />
                       {!readonly
                         ? (
                           <input
@@ -260,6 +282,20 @@ const ConfigVar: FC<IConfigVarProps> = ({ promptVariables, readonly, onPromptVar
         />
       )}
 
+      {isShowDeleteContextVarModal && (
+        <ConfirmModal
+          isShow={isShowDeleteContextVarModal}
+          title={t('appDebug.feature.dataSet.queryVariable.deleteContextVarTitle', { varName: promptVariables[removeIndex as number]?.name })}
+          desc={t('appDebug.feature.dataSet.queryVariable.deleteContextVarTip') as string}
+          confirmBtnClassName='bg-[#B42318] hover:bg-[#B42318]'
+          onConfirm={() => {
+            didRemoveVar(removeIndex as number)
+            hideDeleteContextVarModal()
+          }}
+          onCancel={hideDeleteContextVarModal}
+        />
+      )}
+
     </Panel>
   )
 }

web/app/components/app/configuration/config-var/input-type-icon.tsx

@@ -1,31 +1,25 @@
 'use client'
 import React from 'react'
 import type { FC } from 'react'
+import { Paragraph, TypeSquare } from '@/app/components/base/icons/src/vender/solid/editor'
+import { CheckDone01 } from '@/app/components/base/icons/src/vender/solid/general'
 
 export type IInputTypeIconProps = {
   type: 'string' | 'select'
+  className: string
 }
 
-const IconMap = (type: IInputTypeIconProps['type']) => {
+const IconMap = (type: IInputTypeIconProps['type'], className: string) => {
+  const classNames = `w-3.5 h-3.5 ${className}`
   const icons = {
     string: (
-      <svg width="12" height="12" viewBox="0 0 12 12" fill="none" xmlns="http://www.w3.org/2000/svg">
-        <path fillRule="evenodd" clipRule="evenodd" d="M3.52593 0.166672H8.47411C8.94367 0.166665 9.33123 0.166659 9.64692 0.192452C9.97481 0.219242 10.2762 0.276738 10.5593 0.420991C10.9984 0.644695 11.3553 1.00165 11.579 1.44069C11.7233 1.72381 11.7808 2.02522 11.8076 2.35311C11.8334 2.6688 11.8334 3.05634 11.8334 3.5259V8.47411C11.8334 8.94367 11.8334 9.33121 11.8076 9.6469C11.7808 9.97479 11.7233 10.2762 11.579 10.5593C11.3553 10.9984 10.9984 11.3553 10.5593 11.579C10.2762 11.7233 9.97481 11.7808 9.64692 11.8076C9.33123 11.8334 8.94369 11.8333 8.47413 11.8333H3.52592C3.05636 11.8333 2.66882 11.8334 2.35312 11.8076C2.02523 11.7808 1.72382 11.7233 1.44071 11.579C1.00167 11.3553 0.644711 10.9984 0.421006 10.5593C0.276753 10.2762 0.219257 9.97479 0.192468 9.6469C0.166674 9.33121 0.16668 8.94366 0.166687 8.4741V3.52591C0.16668 3.05635 0.166674 2.6688 0.192468 2.35311C0.219257 2.02522 0.276753 1.72381 0.421006 1.44069C0.644711 1.00165 1.00167 0.644695 1.44071 0.420991C1.72382 0.276738 2.02523 0.219242 2.35312 0.192452C2.66882 0.166659 3.05637 0.166665 3.52593 0.166672ZM3.08335 3.08334C3.08335 2.76117 3.34452 2.50001 3.66669 2.50001H8.33335C8.65552 2.50001 8.91669 2.76117 8.91669 3.08334C8.91669 3.4055 8.65552 3.66667 8.33335 3.66667H6.58335V8.91667C6.58335 9.23884 6.32219 9.5 6.00002 9.5C5.67785 9.5 5.41669 9.23884 5.41669 8.91667V3.66667H3.66669C3.34452 3.66667 3.08335 3.4055 3.08335 3.08334Z" fill="#98A2B3" />
-      </svg>
+      <TypeSquare className={classNames} />
     ),
     paragraph: (
-      <svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg">
-        <path fillRule="evenodd" clipRule="evenodd" d="M1.16669 5.83329C1.16669 5.51113 1.42785 5.24996 1.75002 5.24996H9.33335C9.65552 5.24996 9.91669 5.51113 9.91669 5.83329C9.91669 6.15546 9.65552 6.41663 9.33335 6.41663H1.75002C1.42785 6.41663 1.16669 6.15546 1.16669 5.83329Z" fill="#98A2B3"/>
-        <path fillRule="evenodd" clipRule="evenodd" d="M1.16669 3.49996C1.16669 3.17779 1.42785 2.91663 1.75002 2.91663H11.6667C11.9889 2.91663 12.25 3.17779 12.25 3.49996C12.25 3.82213 11.9889 4.08329 11.6667 4.08329H1.75002C1.42785 4.08329 1.16669 3.82213 1.16669 3.49996Z" fill="#98A2B3"/>
-        <path fillRule="evenodd" clipRule="evenodd" d="M1.16669 8.16663C1.16669 7.84446 1.42785 7.58329 1.75002 7.58329H11.6667C11.9889 7.58329 12.25 7.84446 12.25 8.16663C12.25 8.48879 11.9889 8.74996 11.6667 8.74996H1.75002C1.42785 8.74996 1.16669 8.48879 1.16669 8.16663Z" fill="#98A2B3"/>
-        <path fillRule="evenodd" clipRule="evenodd" d="M1.16669 10.5C1.16669 10.1778 1.42785 9.91663 1.75002 9.91663H9.33335C9.65552 9.91663 9.91669 10.1778 9.91669 10.5C9.91669 10.8221 9.65552 11.0833 9.33335 11.0833H1.75002C1.42785 11.0833 1.16669 10.8221 1.16669 10.5Z" fill="#98A2B3"/>
-      </svg>
+      <Paragraph className={classNames} />
     ),
     select: (
-      <svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg">
-        <path fillRule="evenodd" clipRule="evenodd" d="M7.48913 4.08334H3.01083C2.70334 4.08333 2.43804 4.08333 2.21955 4.10118C1.98893 4.12002 1.75955 4.16162 1.53883 4.27408C1.20955 4.44186 0.941831 4.70958 0.774053 5.03886C0.66159 5.25958 0.619989 5.48896 0.601147 5.71958C0.583295 5.93807 0.583304 6.20334 0.583313 6.51084V10.9892C0.583304 11.2967 0.583295 11.5619 0.601147 11.7804C0.619989 12.0111 0.66159 12.2404 0.774053 12.4612C0.941831 12.7904 1.20955 13.0582 1.53883 13.2259C1.75955 13.3384 1.98893 13.38 2.21955 13.3988C2.43803 13.4167 2.70329 13.4167 3.01077 13.4167H7.48912C7.7966 13.4167 8.06193 13.4167 8.28041 13.3988C8.51103 13.38 8.74041 13.3384 8.96113 13.2259C9.29041 13.0582 9.55813 12.7904 9.72591 12.4612C9.83837 12.2404 9.87997 12.0111 9.89882 11.7804C9.91667 11.5619 9.91666 11.2967 9.91665 10.9892V6.51087C9.91666 6.20336 9.91667 5.93808 9.89882 5.71958C9.87997 5.48896 9.83837 5.25958 9.72591 5.03886C9.55813 4.70958 9.29041 4.44186 8.96113 4.27408C8.74041 4.16162 8.51103 4.12002 8.28041 4.10118C8.06192 4.08333 7.79663 4.08333 7.48913 4.08334ZM7.70413 7.70416C7.93193 7.47635 7.93193 7.107 7.70413 6.8792C7.47632 6.65139 7.10697 6.65139 6.87917 6.8792L4.66665 9.09172L3.91246 8.33753C3.68465 8.10973 3.31531 8.10973 3.0875 8.33753C2.8597 8.56534 2.8597 8.93468 3.0875 9.16249L4.25417 10.3292C4.48197 10.557 4.85132 10.557 5.07913 10.3292L7.70413 7.70416Z" fill="#98A2B3" />
-        <path d="M10.9891 0.583344H6.51083C6.20334 0.583334 5.93804 0.583326 5.71955 0.601177C5.48893 0.620019 5.25955 0.66162 5.03883 0.774084C4.70955 0.941862 4.44183 1.20958 4.27405 1.53886C4.16159 1.75958 4.11999 1.98896 4.10115 2.21958C4.08514 2.41545 4.08349 2.64892 4.08333 2.91669L7.51382 2.91668C7.79886 2.91662 8.10791 2.91654 8.37541 2.9384C8.67818 2.96314 9.07818 3.02436 9.49078 3.23459C10.0396 3.51422 10.4858 3.96041 10.7654 4.50922C10.9756 4.92182 11.0369 5.32182 11.0616 5.62459C11.0835 5.8921 11.0834 6.20115 11.0833 6.48619L11.0833 9.91666C11.3511 9.9165 11.5845 9.91485 11.7804 9.89885C12.011 9.88 12.2404 9.8384 12.4611 9.72594C12.7904 9.55816 13.0581 9.29045 13.2259 8.96116C13.3384 8.74044 13.38 8.51106 13.3988 8.28044C13.4167 8.06196 13.4167 7.7967 13.4166 7.48922V3.01087C13.4167 2.70339 13.4167 2.43807 13.3988 2.21958C13.38 1.98896 13.3384 1.75958 13.2259 1.53886C13.0581 1.20958 12.7904 0.941862 12.4611 0.774084C12.2404 0.66162 12.011 0.620019 11.7804 0.601177C11.5619 0.583326 11.2966 0.583334 10.9891 0.583344Z" fill="#98A2B3" />
-      </svg>
+      <CheckDone01 className={classNames} />
     ),
   }
 
@@ -34,8 +28,9 @@ const IconMap = (type: IInputTypeIconProps['type']) => {
 
 const InputTypeIcon: FC<IInputTypeIconProps> = ({
   type,
+  className,
 }) => {
-  const Icon = IconMap(type)
+  const Icon = IconMap(type, className)
   return Icon
 }
 

web/app/components/app/configuration/config/automatic/get-automatic-res.tsx

@@ -14,6 +14,8 @@ import OpeningStatement from '@/app/components/app/configuration/features/chat-g
 import GroupName from '@/app/components/app/configuration/base/group-name'
 import Loading from '@/app/components/base/loading'
 import Confirm from '@/app/components/base/confirm'
+// type
+import type { AutomaticRes } from '@/service/debug'
 
 const noDataIcon = (
   <svg width="56" height="56" viewBox="0 0 56 56" fill="none" xmlns="http://www.w3.org/2000/svg">
@@ -21,12 +23,6 @@ const noDataIcon = (
   </svg>
 )
 
-export type AutomaticRes = {
-  prompt: string
-  variables: string[]
-  opening_statement: string
-}
-
 export type IGetAutomaticResProps = {
   mode: AppType
   isShow: boolean
@@ -98,7 +94,7 @@ const GetAutomaticRes: FC<IGetAutomaticResProps> = ({
         audiences,
         hoping_to_solve: hopingToSolve,
       })
-      setRes(res as AutomaticRes)
+      setRes(res)
     }
     finally {
       setLoadingFalse()
@@ -193,7 +189,7 @@ const GetAutomaticRes: FC<IGetAutomaticResProps> = ({
             onClose={() => setShowConfirmOverwrite(false)}
             onConfirm={() => {
               setShowConfirmOverwrite(false)
-              onFinished(res as AutomaticRes)
+              onFinished(res!)
             }}
             onCancel={() => setShowConfirmOverwrite(false)}
           />

web/app/components/app/configuration/dataset-config/context-var/index.tsx

@@ -0,0 +1,39 @@
+'use client'
+import type { FC } from 'react'
+import React from 'react'
+import { useTranslation } from 'react-i18next'
+import cn from 'classnames'
+import type { Props } from './var-picker'
+import VarPicker from './var-picker'
+import { BracketsX } from '@/app/components/base/icons/src/vender/line/development'
+import Tooltip from '@/app/components/base/tooltip'
+import { HelpCircle } from '@/app/components/base/icons/src/vender/line/general'
+
+const ContextVar: FC<Props> = (props) => {
+  const { t } = useTranslation()
+  const { value, options } = props
+  const currItem = options.find(item => item.value === value)
+  const notSetVar = !currItem
+  return (
+    <div className={cn(notSetVar ? 'rounded-bl-xl rounded-br-xl bg-[#FEF0C7] border-[#FEF0C7]' : 'border-gray-200', 'flex justify-between items-center h-12 px-3 border-t ')}>
+      <div className='flex items-center space-x-1 shrink-0'>
+        <div className='p-1'>
+          <BracketsX className='w-4 h-4 text-primary-500'/>
+        </div>
+        <div className='mr-1 text-sm font-medium text-gray-800'>{t('appDebug.feature.dataSet.queryVariable.title')}</div>
+        <Tooltip
+          htmlContent={<div className='w-[180px]'>
+            {t('appDebug.feature.dataSet.queryVariable.tip')}
+          </div>}
+          selector='context-var-tooltip'
+        >
+          <HelpCircle className='w-3.5 h-3.5 text-gray-400'/>
+        </Tooltip>
+      </div>
+
+      <VarPicker {...props} />
+    </div>
+  )
+}
+
+export default React.memo(ContextVar)

web/app/components/app/configuration/dataset-config/context-var/style.module.css

@@ -0,0 +1,3 @@
+.trigger:hover .dropdownIcon {
+  color: #98A2B3;
+}

web/app/components/app/configuration/dataset-config/context-var/var-picker.tsx

@@ -0,0 +1,99 @@
+'use client'
+import type { FC } from 'react'
+import React, { useState } from 'react'
+import { useTranslation } from 'react-i18next'
+import { ChevronDownIcon } from '@heroicons/react/24/outline'
+import cn from 'classnames'
+import s from './style.module.css'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+  PortalToFollowElemTrigger,
+} from '@/app/components/base/portal-to-follow-elem'
+import type { IInputTypeIconProps } from '@/app/components/app/configuration/config-var/input-type-icon'
+import IconTypeIcon from '@/app/components/app/configuration/config-var/input-type-icon'
+
+type Option = { name: string; value: string; type: string }
+export type Props = {
+  value: string | undefined
+  options: Option[]
+  onChange: (value: string) => void
+}
+
+const VarItem: FC<{ item: Option }> = ({ item }) => (
+  <div className='flex items-center h-[18px] px-1 bg-[#EFF8FF] rounded space-x-1'>
+    <IconTypeIcon type={item.type as IInputTypeIconProps['type']} className='text-[#1570EF]' />
+    <div className='flex text-xs font-medium text-[#1570EF]'>
+      <span className='opacity-60'>{'{{'}</span>
+      <span className='max-w-[150px] truncate'>{item.value}</span>
+      <span className='opacity-60'>{'}}'}</span>
+    </div>
+  </div>
+)
+const VarPicker: FC<Props> = ({
+  value,
+  options,
+  onChange,
+}) => {
+  const { t } = useTranslation()
+  const [open, setOpen] = useState(false)
+  const currItem = options.find(item => item.value === value)
+  const notSetVar = !currItem
+  return (
+    <PortalToFollowElem
+      open={open}
+      onOpenChange={setOpen}
+      placement='bottom-end'
+      offset={{
+        mainAxis: 8,
+      }}
+    >
+      <PortalToFollowElemTrigger onClick={() => setOpen(v => !v)}>
+        <div className={cn(
+          s.trigger,
+          notSetVar ? 'bg-[#FFFCF5] border-[#FEDF89] text-[#DC6803]' : ' hover:bg-gray-50 border-gray-200 text-primary-600',
+          open ? 'bg-gray-50' : 'bg-white',
+          `
+          flex items-center h-8 justify-center px-2 space-x-1 rounded-lg border  shadow-xs cursor-pointer
+          text-[13px]  font-medium
+          `)}>
+          <div>
+            {value
+              ? (
+                <VarItem item={currItem as Option} />
+              )
+              : (<div>
+                {t('appDebug.feature.dataSet.queryVariable.choosePlaceholder')}
+              </div>)}
+          </div>
+          <ChevronDownIcon className={cn(s.dropdownIcon, open && 'rotate-180 text-[#98A2B3]', 'w-3.5 h-3.5')} />
+        </div>
+      </PortalToFollowElemTrigger>
+      <PortalToFollowElemContent style={{ zIndex: 1000 }}>
+        {options.length > 0
+          ? (<div className='w-[240px] max-h-[50vh] overflow-y-auto p-1  border bg-white border-gray-200 rounded-lg shadow-lg'>
+            {options.map(({ name, value, type }, index) => (
+              <div
+                key={index}
+                className='px-3 py-1 flex rounded-lg hover:bg-gray-50 cursor-pointer'
+                onClick={() => {
+                  onChange(value)
+                  setOpen(false)
+                }}
+              >
+                <VarItem item={{ name, value, type }} />
+              </div>
+            ))}
+          </div>)
+          : (
+            <div className='w-[240px] p-6 bg-white border border-gray-200 rounded-lg shadow-lg'>
+              <div className='mb-1 text-sm font-medium text-gray-700'>{t('appDebug.feature.dataSet.queryVariable.noVar')}</div>
+              <div className='text-xs leading-normal text-gray-500'>{t('appDebug.feature.dataSet.queryVariable.noVarTip')}</div>
+            </div>
+          )}
+
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+export default React.memo(VarPicker)

web/app/components/app/configuration/dataset-config/index.tsx

@@ -10,8 +10,10 @@ import FeaturePanel from '../base/feature-panel'
 import OperationBtn from '../base/operation-btn'
 import CardItem from './card-item'
 import SelectDataSet from './select-dataset'
+import ContextVar from './context-var'
 import ConfigContext from '@/context/debug-configuration'
 import type { DataSet } from '@/models/datasets'
+import { AppType } from '@/types/app'
 
 const Icon = (
   <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
@@ -23,9 +25,12 @@ const Icon = (
 const DatasetConfig: FC = () => {
   const { t } = useTranslation()
   const {
+    mode,
     dataSets: dataSet,
     setDataSets: setDataSet,
     setFormattingChanged,
+    modelConfig,
+    setModelConfig,
   } = useContext(ConfigContext)
   const selectedIds = dataSet.map(item => item.id)
 
@@ -60,6 +65,25 @@ const DatasetConfig: FC = () => {
     setFormattingChanged(true)
   }
 
+  const promptVariables = modelConfig.configs.prompt_variables
+  const promptVariablesToSelect = promptVariables.map(item => ({
+    name: item.name,
+    type: item.type,
+    value: item.key,
+  }))
+  const selectedContextVar = promptVariables?.find(item => item.is_context_var)
+  const handleSelectContextVar = (selectedValue: string) => {
+    const newModelConfig = produce(modelConfig, (draft) => {
+      draft.configs.prompt_variables = modelConfig.configs.prompt_variables.map((item) => {
+        return ({
+          ...item,
+          is_context_var: item.key === selectedValue,
+        })
+      })
+    })
+    setModelConfig(newModelConfig)
+  }
+
   return (
     <FeaturePanel
       className='mt-3'
@@ -67,10 +91,11 @@ const DatasetConfig: FC = () => {
       title={t('appDebug.feature.dataSet.title')}
       headerRight={<OperationBtn type="add" onClick={showSelectDataSet} />}
       hasHeaderBottomBorder={!hasData}
+      noBodySpacing
     >
       {hasData
         ? (
-          <div className='flex flex-wrap justify-between'>
+          <div className='flex flex-wrap mt-1 px-3 justify-between'>
             {dataSet.map(item => (
               <CardItem
                 className="mb-2"
@@ -82,9 +107,19 @@ const DatasetConfig: FC = () => {
           </div>
         )
         : (
-          <div className='pt-2 pb-1 text-xs text-gray-500'>{t('appDebug.feature.dataSet.noData')}</div>
+          <div className='mt-1 px-3 pb-3'>
+            <div className='pt-2 pb-1 text-xs text-gray-500'>{t('appDebug.feature.dataSet.noData')}</div>
+          </div>
         )}
 
+      {mode === AppType.completion && dataSet.length > 0 && (
+        <ContextVar
+          value={selectedContextVar?.key}
+          options={promptVariablesToSelect}
+          onChange={handleSelectContextVar}
+        />
+      )}
+
       {isShowSelectDataSet && (
         <SelectDataSet
           isShow={isShowSelectDataSet}

web/app/components/app/configuration/debug/index.tsx

@@ -10,6 +10,7 @@ import dayjs from 'dayjs'
 import HasNotSetAPIKEY from '../base/warning-mask/has-not-set-api'
 import FormattingChanged from '../base/warning-mask/formatting-changed'
 import GroupName from '../base/group-name'
+import CannotQueryDataset from '../base/warning-mask/cannot-query-dataset'
 import { AppType } from '@/types/app'
 import PromptValuePanel, { replaceStringWithValues } from '@/app/components/app/configuration/prompt-value-panel'
 import type { IChatItem } from '@/app/components/app/chat/type'
@@ -23,7 +24,6 @@ import { promptVariablesToUserInputsForm } from '@/utils/model-config'
 import TextGeneration from '@/app/components/app/text-generate/item'
 import { IS_CE_EDITION } from '@/config'
 import { useProviderContext } from '@/context/provider-context'
-
 type IDebug = {
   hasSetAPIKEY: boolean
   onSetting: () => void
@@ -52,6 +52,7 @@ const Debug: FC<IDebug> = ({
     dataSets,
     modelConfig,
     completionParams,
+    hasSetContextVar,
   } = useContext(ConfigContext)
   const { speech2textDefaultModel } = useProviderContext()
   const [chatList, setChatList, getChatList] = useGetState<IChatItem[]>([])
@@ -77,6 +78,7 @@ const Debug: FC<IDebug> = ({
   const [isResponsing, { setTrue: setResponsingTrue, setFalse: setResponsingFalse }] = useBoolean(false)
   const [abortController, setAbortController] = useState<AbortController | null>(null)
   const [isShowFormattingChangeConfirm, setIsShowFormattingChangeConfirm] = useState(false)
+  const [isShowCannotQueryDataset, setShowCannotQueryDataset] = useState(false)
   const [isShowSuggestion, setIsShowSuggestion] = useState(false)
   const [messageTaskId, setMessageTaskId] = useState('')
   const [hasStopResponded, setHasStopResponded, getHasStopResponded] = useGetState(false)
@@ -157,6 +159,7 @@ const Debug: FC<IDebug> = ({
     const postModelConfig: BackendModelConfig = {
       pre_prompt: modelConfig.configs.prompt_template,
       user_input_form: promptVariablesToUserInputsForm(modelConfig.configs.prompt_variables),
+      dataset_query_variable: '',
       opening_statement: introduction,
       more_like_this: {
         enabled: false,
@@ -298,6 +301,7 @@ const Debug: FC<IDebug> = ({
   }, [controlClearChatMessage])
 
   const [completionRes, setCompletionRes] = useState('')
+  const [messageId, setMessageId] = useState<string | null>(null)
 
   const sendTextCompletion = async () => {
     if (isResponsing) {
@@ -305,6 +309,11 @@ const Debug: FC<IDebug> = ({
       return false
     }
 
+    if (dataSets.length > 0 && !hasSetContextVar) {
+      setShowCannotQueryDataset(true)
+      return true
+    }
+
     if (!checkCanSend())
       return
 
@@ -314,10 +323,12 @@ const Debug: FC<IDebug> = ({
         id,
       },
     }))
+    const contextVar = modelConfig.configs.prompt_variables.find(item => item.is_context_var)?.key
 
     const postModelConfig: BackendModelConfig = {
       pre_prompt: modelConfig.configs.prompt_template,
       user_input_form: promptVariablesToUserInputsForm(modelConfig.configs.prompt_variables),
+      dataset_query_variable: contextVar || '',
       opening_statement: introduction,
       suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
       speech_to_text: speechToTextConfig,
@@ -340,13 +351,15 @@ const Debug: FC<IDebug> = ({
     }
 
     setCompletionRes('')
+    setMessageId('')
     const res: string[] = []
 
     setResponsingTrue()
     sendCompletionMessage(appId, data, {
-      onData: (data: string) => {
+      onData: (data: string, _isFirstMessage: boolean, { messageId }) => {
         res.push(data)
         setCompletionRes(res.join(''))
+        setMessageId(messageId)
       },
       onCompleted() {
         setResponsingFalse()
@@ -415,6 +428,7 @@ const Debug: FC<IDebug> = ({
                 content={completionRes}
                 isLoading={!completionRes && isResponsing}
                 isInstalledApp={false}
+                messageId={messageId}
               />
             )}
           </div>
@@ -425,6 +439,11 @@ const Debug: FC<IDebug> = ({
             onCancel={handleCancel}
           />
         )}
+        {isShowCannotQueryDataset && (
+          <CannotQueryDataset
+            onConfirm={() => setShowCannotQueryDataset(false)}
+          />
+        )}
       </div>
 
       {!hasSetAPIKEY && (<HasNotSetAPIKEY isTrailFinished={!IS_CE_EDITION} onSetting={onSetting} />)}