feat: bump version to 0.3.22 (#1153)

Co-authored-by: jyong <jyong@dify.ai>
Co-authored-by: StyleZhang <jasonapring2015@outlook.com>

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -0,0 +1,49 @@
+name: "🕷️ Bug report"
+description: Report errors or unexpected behavior
+labels:
+- bug
+body:
+- type: markdown
+  attributes:
+    value: Please make sure to [search for existing issues](https://github.com/langgenius/dify/issues) before filing a new one!
+- type: input
+  attributes:
+    label: Dify version
+    placeholder: 0.3.21
+    description: See about section in Dify console
+  validations:
+    required: true
+
+- type: dropdown
+  attributes:
+    label: Cloud or Self Hosted
+    description: How / Where was Dify installed from?
+    multiple: true
+    options:
+      - Cloud
+      - Self Hosted
+      - Other (please specify in "Steps to Reproduce")
+  validations:
+    required: true
+
+- type: textarea
+  attributes:
+    label: Steps to reproduce
+    description: We highly suggest including screenshots and a bug report log.
+    placeholder: Having detailed steps helps us reproduce the bug. 
+  validations:
+    required: true
+
+- type: textarea
+  attributes:
+    label: ✔️ Expected Behavior
+    placeholder: What were you expecting?
+  validations:
+    required: false
+
+- type: textarea
+  attributes:
+    label: ❌ Actual Behavior
+    placeholder: What happened instead?
+  validations:
+    required: false

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,8 @@
+blank_issues_enabled: false
+contact_links:
+  - name: "\U0001F4DA Dify user documentation"
+    url: https://docs.dify.ai/getting-started/readme
+    about: Documentation for users of Dify
+  - name: "\U0001F4DA Dify dev documentation"
+    url: https://docs.dify.ai/getting-started/install-self-hosted
+    about: Documentation for people interested in developing and contributing for Dify

.github/ISSUE_TEMPLATE/document_issue.yml

@@ -0,0 +1,11 @@
+name: "📚 Documentation Issue"
+description: Report issues in our documentation
+labels: 
+- ducumentation
+body:
+- type: textarea
+  attributes: 
+    label: Provide a description of requested docs changes
+    placeholder: Briefly describe which document needs to be corrected and why.
+  validations:
+    required: true

.github/ISSUE_TEMPLATE/feature_request.yml

@@ -0,0 +1,26 @@
+name: "⭐ Feature or enhancement request"
+description: Propose something new.
+labels:
+- enhancement
+body:
+- type: textarea
+  attributes: 
+    label: Description of the new feature / enhancement
+    placeholder: What is the expected behavior of the proposed feature?
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Scenario when this would be used?
+    placeholder: What is the scenario this would be used? Why is this important to your workflow as a dify user?
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: Supporting information
+    placeholder: "Having additional evidence, data, tweets, blog posts, research, ... anything is extremely helpful. This information provides context to the scenario that may otherwise be lost."
+  validations:
+    required: false
+- type: markdown
+  attributes:
+    value: Please limit one request per issue.

.github/ISSUE_TEMPLATE/translation_issue.yml

@@ -0,0 +1,46 @@
+name: "🌐 Localization/Translation issue"
+description: Report incorrect translations.
+labels:
+- translation
+body:
+- type: markdown
+  attributes:
+    value: Please make sure to [search for existing issues](https://github.com/langgenius/dify/issues) before filing a new one!
+- type: input
+  attributes:
+    label: Dify version
+    placeholder: 0.3.21
+    description: Hover over system tray icon or look at Settings
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: Utility with translation issue
+    placeholder: Some area
+    description: Please input here the utility with the translation issue
+  validations:
+    required: true
+- type: input
+  attributes:
+    label: 🌐 Language affected
+    placeholder: "German"
+  validations:
+    required: true
+- type: textarea
+  attributes: 
+    label: ❌ Actual phrase(s)
+    placeholder: What is there? Please include a screenshot as that is extremely helpful.
+  validations:
+    required: true
+- type: textarea
+  attributes: 
+    label: ✔️ Expected phrase(s)
+    placeholder: What was expected?
+  validations:
+    required: true
+- type: textarea
+  attributes:
+    label: ℹ Why is the current translation wrong
+    placeholder: Why do you feel this is incorrect?
+  validations:
+    required: true

.github/ISSUE_TEMPLATE/🐛-bug-report.md

@@ -1,32 +0,0 @@
----
-name: "\U0001F41B Bug report"
-about: Create a report to help us improve
-title: ''
-labels: bug
-assignees: ''
-
----
-
-<!--
-  Please provide a clear and concise description of what the bug is. Include
-  screenshots if needed. Please test using the latest version of the relevant
-  Dify packages to make sure your issue has not already been fixed.
--->
-
-Dify version: Cloud | Self Host
-
-## Steps To Reproduce
-<!--
-  Your bug will get fixed much faster if we can run your code and it doesn't
-  have dependencies other than Dify. Issues without reproduction steps or
-  code examples may be immediately closed as not actionable.
--->
-
-1.
-2.
-
-
-## The current behavior
-
-
-## The expected behavior

.github/ISSUE_TEMPLATE/🚀-feature-request.md

@@ -1,20 +0,0 @@
----
-name: "\U0001F680 Feature request"
-about: Suggest an idea for this project
-title: ''
-labels: enhancement
-assignees: ''
-
----
-
-**Is your feature request related to a problem? Please describe.**
-A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
-
-**Describe the solution you'd like**
-A clear and concise description of what you want to happen.
-
-**Describe alternatives you've considered**
-A clear and concise description of any alternative solutions or features you've considered.
-
-**Additional context**
-Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/🤔-questions-and-help.md

@@ -1,10 +0,0 @@
----
-name: "\U0001F914 Questions and Help"
-about: Ask a usage or consultation question
-title: ''
-labels: ''
-assignees: ''
-
----
-
-

.github/workflows/check_no_chinese_comments.py

@@ -20,7 +20,8 @@ def check_file_for_chinese_comments(file_path):
 def main():
     has_chinese = False
     excluded_files = ["model_template.py", 'stopwords.py', 'commands.py',
-                      'indexing_runner.py', 'web_reader_tool.py', 'spark_provider.py']
+                      'indexing_runner.py', 'web_reader_tool.py', 'spark_provider.py',
+                      'prompts.py']
 
     for root, _, files in os.walk("."):
         for file in files:

.gitignore

@@ -149,4 +149,5 @@ sdks/python-client/build
 sdks/python-client/dist
 sdks/python-client/dify_client.egg-info
 
-.vscode/
+.vscode/*
+!.vscode/launch.json

.vscode/launch.json

@@ -0,0 +1,27 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Flask",
+            "type": "python",
+            "request": "launch",
+            "module": "flask",
+            "env": {
+                "FLASK_APP": "api/app.py",
+                "FLASK_DEBUG": "1",
+                "GEVENT_SUPPORT": "True"
+            },
+            "args": [
+                "run",
+                "--host=0.0.0.0",
+                "--port=5001",
+                "--debug"
+            ],
+            "jinja": true,
+            "justMyCode": true
+        }
+    ]
+}

api/Dockerfile

@@ -1,7 +1,18 @@
-FROM python:3.10-slim
+# packages install stage
+FROM python:3.10-slim AS base
 
 LABEL maintainer="takatost@gmail.com"
 
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends gcc g++ python3-dev libc-dev libffi-dev
+
+COPY requirements.txt /requirements.txt
+
+RUN pip install --prefix=/pkg -r requirements.txt
+
+# build stage
+FROM python:3.10-slim AS builder
+
 ENV FLASK_APP app.py
 ENV EDITION SELF_HOSTED
 ENV DEPLOY_ENV PRODUCTION
@@ -15,15 +26,17 @@ EXPOSE 5001
 
 WORKDIR /app/api
 
-RUN apt-get update && \
-    apt-get install -y bash curl wget vim gcc g++ python3-dev libc-dev libffi-dev nodejs
-
-COPY requirements.txt /app/api/requirements.txt
-
-RUN pip install -r requirements.txt
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends bash curl wget vim nodejs \
+    && apt-get autoremove \
+    && rm -rf /var/lib/apt/lists/*
 
+COPY --from=base /pkg /usr/local
 COPY . /app/api/
 
+RUN python -c "from transformers import GPT2TokenizerFast; GPT2TokenizerFast.from_pretrained('gpt2')"
+ENV TRANSFORMERS_OFFLINE true
+
 COPY docker/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 

api/README.md

@@ -52,7 +52,7 @@
    flask run --host 0.0.0.0 --port=5001 --debug
    ```
 7. Setup your application by visiting http://localhost:5001/console/api/setup or other apis...
-8. If you need to debug local async processing, you can run `celery -A app.celery worker -Q dataset,generation,mail`, celery can do dataset importing and other async tasks.
+8. If you need to debug local async processing, you can run `celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail`, celery can do dataset importing and other async tasks.
 
 8. Start frontend
 

api/app.py

@@ -1,6 +1,6 @@
 # -*- coding:utf-8 -*-
 import os
-from datetime import datetime
+from datetime import datetime, timedelta
 
 from werkzeug.exceptions import Forbidden
 
@@ -145,8 +145,12 @@ def load_user(user_id):
                     _create_tenant_for_account(account)
                 session['workspace_id'] = account.current_tenant_id
 
-            account.last_active_at = datetime.utcnow()
-            db.session.commit()
+            current_time = datetime.utcnow()
+
+            # update last_active_at when last_active_at is more than 10 minutes ago
+            if current_time - account.last_active_at > timedelta(minutes=10):
+                account.last_active_at = current_time
+                db.session.commit()
 
             # Log in the user with the updated user_id
             flask_login.login_user(account, remember=True)

api/commands.py

@@ -6,6 +6,7 @@ import string
 import time
 
 import click
+from tqdm import tqdm
 from flask import current_app
 from langchain.embeddings import OpenAIEmbeddings
 from werkzeug.exceptions import NotFound
@@ -21,9 +22,9 @@ from libs.password import password_pattern, valid_password, hash_password
 from libs.helper import email as email_validate
 from extensions.ext_database import db
 from libs.rsa import generate_key_pair
-from models.account import InvitationCode, Tenant
+from models.account import InvitationCode, Tenant, TenantAccountJoin
 from models.dataset import Dataset, DatasetQuery, Document
-from models.model import Account
+from models.model import Account, AppModelConfig, App
 import secrets
 import base64
 
@@ -439,6 +440,107 @@ def update_qdrant_indexes():
 
     click.echo(click.style('Congratulations! Update {} dataset indexes.'.format(create_count), fg='green'))
 
+@click.command('update_app_model_configs', help='Migrate data to support paragraph variable.')
+@click.option("--batch-size", default=500, help="Number of records to migrate in each batch.")
+def update_app_model_configs(batch_size):
+    pre_prompt_template = '{{default_input}}'
+    user_input_form_template = {
+        "en-US": [
+            {
+                "paragraph": {
+                    "label": "Query",
+                    "variable": "default_input",
+                    "required": False,
+                    "default": ""
+                }
+            }
+        ],
+        "zh-Hans": [
+            {
+                "paragraph": {
+                    "label": "查询内容",
+                    "variable": "default_input",
+                    "required": False,
+                    "default": ""
+                }
+            }
+        ]
+    }
+
+    click.secho("Start migrate old data that the text generator can support paragraph variable.", fg='green')
+
+    total_records = db.session.query(AppModelConfig) \
+        .join(App, App.app_model_config_id == AppModelConfig.id) \
+        .filter(App.mode == 'completion') \
+        .count()
+    
+    if total_records == 0:
+        click.secho("No data to migrate.", fg='green')
+        return
+
+    num_batches = (total_records + batch_size - 1) // batch_size
+
+    with tqdm(total=total_records, desc="Migrating Data") as pbar:
+        for i in range(num_batches):
+            offset = i * batch_size
+            limit = min(batch_size, total_records - offset)
+
+            click.secho(f"Fetching batch {i+1}/{num_batches} from source database...", fg='green')
+            
+            data_batch = db.session.query(AppModelConfig) \
+                .join(App, App.app_model_config_id == AppModelConfig.id) \
+                .filter(App.mode == 'completion') \
+                .order_by(App.created_at) \
+                .offset(offset).limit(limit).all()
+            
+            if not data_batch:
+                click.secho("No more data to migrate.", fg='green')
+                break
+
+            try:
+                click.secho(f"Migrating {len(data_batch)} records...", fg='green')
+                for data in data_batch:
+                    # click.secho(f"Migrating data {data.id}, pre_prompt: {data.pre_prompt}, user_input_form: {data.user_input_form}", fg='green')
+
+                    if data.pre_prompt is None:
+                        data.pre_prompt = pre_prompt_template
+                    else:
+                        if pre_prompt_template in data.pre_prompt:
+                            continue
+                        data.pre_prompt += pre_prompt_template
+
+                    app_data = db.session.query(App) \
+                        .filter(App.id == data.app_id) \
+                        .one()
+                    
+                    account_data = db.session.query(Account) \
+                        .join(TenantAccountJoin, Account.id == TenantAccountJoin.account_id) \
+                        .filter(TenantAccountJoin.role == 'owner') \
+                        .filter(TenantAccountJoin.tenant_id == app_data.tenant_id) \
+                        .one_or_none()
+
+                    if not account_data:
+                        continue
+
+                    if data.user_input_form is None or data.user_input_form == 'null':
+                        data.user_input_form = json.dumps(user_input_form_template[account_data.interface_language])
+                    else:
+                        raw_json_data = json.loads(data.user_input_form)
+                        raw_json_data.append(user_input_form_template[account_data.interface_language][0])
+                        data.user_input_form = json.dumps(raw_json_data)
+
+                    # click.secho(f"Updated data {data.id}, pre_prompt: {data.pre_prompt}, user_input_form: {data.user_input_form}", fg='green')
+
+                db.session.commit()
+
+            except Exception as e:
+                click.secho(f"Error while migrating data: {e}, app_id: {data.app_id}, app_model_config_id: {data.id}", fg='red')
+                continue
+            
+            click.secho(f"Successfully migrated batch {i+1}/{num_batches}.", fg='green')
+            
+            pbar.update(len(data_batch))
+
 def register_commands(app):
     app.cli.add_command(reset_password)
     app.cli.add_command(reset_email)
@@ -448,4 +550,5 @@ def register_commands(app):
     app.cli.add_command(sync_anthropic_hosted_providers)
     app.cli.add_command(clean_unused_dataset_indexes)
     app.cli.add_command(create_qdrant_indexes)
-    app.cli.add_command(update_qdrant_indexes)
+    app.cli.add_command(update_qdrant_indexes)
+    app.cli.add_command(update_app_model_configs)

api/config.py

@@ -100,7 +100,7 @@ class Config:
         self.CONSOLE_URL = get_env('CONSOLE_URL')
         self.API_URL = get_env('API_URL')
         self.APP_URL = get_env('APP_URL')
-        self.CURRENT_VERSION = "0.3.19"
+        self.CURRENT_VERSION = "0.3.22"
         self.COMMIT_SHA = get_env('COMMIT_SHA')
         self.EDITION = "SELF_HOSTED"
         self.DEPLOY_ENV = get_env('DEPLOY_ENV')

api/constants/model_template.py

@@ -38,7 +38,18 @@ model_templates = {
                     "presence_penalty": 0,
                     "frequency_penalty": 0
                 }
-            })
+            }),
+            'user_input_form': json.dumps([
+                {
+                    "paragraph": {
+                        "label": "Query",
+                        "variable": "query",
+                        "required": True,
+                        "default": ""
+                    }
+                }
+            ]),
+            'pre_prompt': '{{query}}'
         }
     },
 

api/controllers/console/app/app.py

@@ -29,6 +29,7 @@ model_config_fields = {
     'suggested_questions': fields.Raw(attribute='suggested_questions_list'),
     'suggested_questions_after_answer': fields.Raw(attribute='suggested_questions_after_answer_dict'),
     'speech_to_text': fields.Raw(attribute='speech_to_text_dict'),
+    'retriever_resource': fields.Raw(attribute='retriever_resource_dict'),
     'more_like_this': fields.Raw(attribute='more_like_this_dict'),
     'sensitive_word_avoidance': fields.Raw(attribute='sensitive_word_avoidance_dict'),
     'model': fields.Raw(attribute='model_dict'),

api/controllers/console/app/completion.py

@@ -39,9 +39,10 @@ class CompletionMessageApi(Resource):
 
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, location='json')
-        parser.add_argument('query', type=str, location='json')
+        parser.add_argument('query', type=str, location='json', default='')
         parser.add_argument('model_config', type=dict, required=True, location='json')
         parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
+        parser.add_argument('retriever_from', type=str, required=False, default='dev', location='json')
         args = parser.parse_args()
 
         streaming = args['response_mode'] != 'blocking'
@@ -115,6 +116,7 @@ class ChatMessageApi(Resource):
         parser.add_argument('model_config', type=dict, required=True, location='json')
         parser.add_argument('conversation_id', type=uuid_value, location='json')
         parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
+        parser.add_argument('retriever_from', type=str, required=False, default='dev', location='json')
         args = parser.parse_args()
 
         streaming = args['response_mode'] != 'blocking'

api/controllers/console/auth/activate.py

@@ -16,26 +16,25 @@ from services.account_service import RegisterService
 class ActivateCheckApi(Resource):
     def get(self):
         parser = reqparse.RequestParser()
-        parser.add_argument('workspace_id', type=str, required=True, nullable=False, location='args')
-        parser.add_argument('email', type=email, required=True, nullable=False, location='args')
+        parser.add_argument('workspace_id', type=str, required=False, nullable=True, location='args')
+        parser.add_argument('email', type=email, required=False, nullable=True, location='args')
         parser.add_argument('token', type=str, required=True, nullable=False, location='args')
         args = parser.parse_args()
 
-        account = RegisterService.get_account_if_token_valid(args['workspace_id'], args['email'], args['token'])
+        workspaceId = args['workspace_id']
+        reg_email = args['email']
+        token = args['token']
 
-        tenant = db.session.query(Tenant).filter(
-            Tenant.id == args['workspace_id'],
-            Tenant.status == 'normal'
-        ).first()
+        invitation = RegisterService.get_invitation_if_token_valid(workspaceId, reg_email, token)
 
-        return {'is_valid': account is not None, 'workspace_name': tenant.name}
+        return {'is_valid': invitation is not None, 'workspace_name': invitation['tenant'].name if invitation else None}
 
 
 class ActivateApi(Resource):
     def post(self):
         parser = reqparse.RequestParser()
-        parser.add_argument('workspace_id', type=str, required=True, nullable=False, location='json')
-        parser.add_argument('email', type=email, required=True, nullable=False, location='json')
+        parser.add_argument('workspace_id', type=str, required=False, nullable=True, location='json')
+        parser.add_argument('email', type=email, required=False, nullable=True, location='json')
         parser.add_argument('token', type=str, required=True, nullable=False, location='json')
         parser.add_argument('name', type=str_len(30), required=True, nullable=False, location='json')
         parser.add_argument('password', type=valid_password, required=True, nullable=False, location='json')
@@ -44,12 +43,13 @@ class ActivateApi(Resource):
         parser.add_argument('timezone', type=timezone, required=True, nullable=False, location='json')
         args = parser.parse_args()
 
-        account = RegisterService.get_account_if_token_valid(args['workspace_id'], args['email'], args['token'])
-        if account is None:
+        invitation = RegisterService.get_invitation_if_token_valid(args['workspace_id'], args['email'], args['token'])
+        if invitation is None:
             raise AlreadyActivateError()
 
         RegisterService.revoke_token(args['workspace_id'], args['email'], args['token'])
 
+        account = invitation['account']
         account.name = args['name']
 
         # generate password salt

api/controllers/console/datasets/file.py

@@ -26,7 +26,7 @@ from models.model import UploadFile
 
 cache = TTLCache(maxsize=None, ttl=30)
 
-ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx']
+ALLOWED_EXTENSIONS = ['txt', 'markdown', 'md', 'pdf', 'html', 'htm', 'xlsx', 'docx', 'csv']
 PREVIEW_WORDS_LIMIT = 3000
 
 

api/controllers/console/explore/completion.py

@@ -31,8 +31,9 @@ class CompletionApi(InstalledAppResource):
 
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, location='json')
-        parser.add_argument('query', type=str, location='json')
+        parser.add_argument('query', type=str, location='json', default='')
         parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
+        parser.add_argument('retriever_from', type=str, required=False, default='explore_app', location='json')
         args = parser.parse_args()
 
         streaming = args['response_mode'] == 'streaming'
@@ -92,6 +93,7 @@ class ChatApi(InstalledAppResource):
         parser.add_argument('query', type=str, required=True, location='json')
         parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
         parser.add_argument('conversation_id', type=uuid_value, location='json')
+        parser.add_argument('retriever_from', type=str, required=False, default='explore_app', location='json')
         args = parser.parse_args()
 
         streaming = args['response_mode'] == 'streaming'

api/controllers/console/explore/message.py

@@ -30,6 +30,25 @@ class MessageListApi(InstalledAppResource):
         'rating': fields.String
     }
 
+    retriever_resource_fields = {
+        'id': fields.String,
+        'message_id': fields.String,
+        'position': fields.Integer,
+        'dataset_id': fields.String,
+        'dataset_name': fields.String,
+        'document_id': fields.String,
+        'document_name': fields.String,
+        'data_source_type': fields.String,
+        'segment_id': fields.String,
+        'score': fields.Float,
+        'hit_count': fields.Integer,
+        'word_count': fields.Integer,
+        'segment_position': fields.Integer,
+        'index_node_hash': fields.String,
+        'content': fields.String,
+        'created_at': TimestampField
+    }
+
     message_fields = {
         'id': fields.String,
         'conversation_id': fields.String,
@@ -37,6 +56,7 @@ class MessageListApi(InstalledAppResource):
         'query': fields.String,
         'answer': fields.String,
         'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
+        'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
         'created_at': TimestampField
     }
 

api/controllers/console/explore/parameter.py

@@ -24,6 +24,7 @@ class AppParameterApi(InstalledAppResource):
         'suggested_questions': fields.Raw,
         'suggested_questions_after_answer': fields.Raw,
         'speech_to_text': fields.Raw,
+        'retriever_resource': fields.Raw,
         'more_like_this': fields.Raw,
         'user_input_form': fields.Raw,
     }
@@ -39,6 +40,7 @@ class AppParameterApi(InstalledAppResource):
             'suggested_questions': app_model_config.suggested_questions_list,
             'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
             'speech_to_text': app_model_config.speech_to_text_dict,
+            'retriever_resource': app_model_config.retriever_resource_dict,
             'more_like_this': app_model_config.more_like_this_dict,
             'user_input_form': app_model_config.user_input_form_list
         }

api/controllers/console/universal_chat/chat.py

@@ -29,9 +29,11 @@ class UniversalChatApi(UniversalChatResource):
         parser.add_argument('provider', type=str, required=True, location='json')
         parser.add_argument('model', type=str, required=True, location='json')
         parser.add_argument('tools', type=list, required=True, location='json')
+        parser.add_argument('retriever_from', type=str, required=False, default='universal_app', location='json')
         args = parser.parse_args()
 
         app_model_config = app_model.app_model_config
+        app_model_config
 
         # update app model config
         args['model_config'] = app_model_config.to_dict()

api/controllers/console/universal_chat/message.py

@@ -36,6 +36,25 @@ class UniversalChatMessageListApi(UniversalChatResource):
         'created_at': TimestampField
     }
 
+    retriever_resource_fields = {
+        'id': fields.String,
+        'message_id': fields.String,
+        'position': fields.Integer,
+        'dataset_id': fields.String,
+        'dataset_name': fields.String,
+        'document_id': fields.String,
+        'document_name': fields.String,
+        'data_source_type': fields.String,
+        'segment_id': fields.String,
+        'score': fields.Float,
+        'hit_count': fields.Integer,
+        'word_count': fields.Integer,
+        'segment_position': fields.Integer,
+        'index_node_hash': fields.String,
+        'content': fields.String,
+        'created_at': TimestampField
+    }
+
     message_fields = {
         'id': fields.String,
         'conversation_id': fields.String,
@@ -43,6 +62,7 @@ class UniversalChatMessageListApi(UniversalChatResource):
         'query': fields.String,
         'answer': fields.String,
         'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
+        'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
         'created_at': TimestampField,
         'agent_thoughts': fields.List(fields.Nested(agent_thought_fields))
     }

api/controllers/console/universal_chat/parameter.py

@@ -1,4 +1,6 @@
 # -*- coding:utf-8 -*-
+import json
+
 from flask_restful import marshal_with, fields
 
 from controllers.console import api
@@ -14,6 +16,7 @@ class UniversalChatParameterApi(UniversalChatResource):
         'suggested_questions': fields.Raw,
         'suggested_questions_after_answer': fields.Raw,
         'speech_to_text': fields.Raw,
+        'retriever_resource': fields.Raw,
     }
 
     @marshal_with(parameters_fields)
@@ -21,12 +24,14 @@ class UniversalChatParameterApi(UniversalChatResource):
         """Retrieve app parameters."""
         app_model = universal_app
         app_model_config = app_model.app_model_config
+        app_model_config.retriever_resource = json.dumps({'enabled': True})
 
         return {
             'opening_statement': app_model_config.opening_statement,
             'suggested_questions': app_model_config.suggested_questions_list,
             'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
             'speech_to_text': app_model_config.speech_to_text_dict,
+            'retriever_resource': app_model_config.retriever_resource_dict,
         }
 
 

api/controllers/console/universal_chat/wraps.py

@@ -47,6 +47,7 @@ def universal_chat_app_required(view=None):
                     suggested_questions=json.dumps([]),
                     suggested_questions_after_answer=json.dumps({'enabled': True}),
                     speech_to_text=json.dumps({'enabled': True}),
+                    retriever_resource=json.dumps({'enabled': True}),
                     more_like_this=None,
                     sensitive_word_avoidance=None,
                     model=json.dumps({

api/controllers/console/workspace/members.py

@@ -72,7 +72,7 @@ class MemberInviteEmailApi(Resource):
                 invitation_results.append({
                     'status': 'success',
                     'email': invitee_email,
-                    'url': f'{console_web_url}/activate?workspace_id={current_user.current_tenant_id}&email={invitee_email}&token={token}'
+                    'url': f'{console_web_url}/activate?email={invitee_email}&token={token}'
                 })
                 account = marshal(account, account_fields)
                 account['role'] = role

api/controllers/service_api/app/app.py

@@ -25,6 +25,7 @@ class AppParameterApi(AppApiResource):
         'suggested_questions': fields.Raw,
         'suggested_questions_after_answer': fields.Raw,
         'speech_to_text': fields.Raw,
+        'retriever_resource': fields.Raw,
         'more_like_this': fields.Raw,
         'user_input_form': fields.Raw,
     }
@@ -39,6 +40,7 @@ class AppParameterApi(AppApiResource):
             'suggested_questions': app_model_config.suggested_questions_list,
             'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
             'speech_to_text': app_model_config.speech_to_text_dict,
+            'retriever_resource': app_model_config.retriever_resource_dict,
             'more_like_this': app_model_config.more_like_this_dict,
             'user_input_form': app_model_config.user_input_form_list
         }

api/controllers/service_api/app/completion.py

@@ -27,9 +27,11 @@ class CompletionApi(AppApiResource):
 
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, location='json')
-        parser.add_argument('query', type=str, location='json')
+        parser.add_argument('query', type=str, location='json', default='')
         parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
         parser.add_argument('user', type=str, location='json')
+        parser.add_argument('retriever_from', type=str, required=False, default='dev', location='json')
+
         args = parser.parse_args()
 
         streaming = args['response_mode'] == 'streaming'
@@ -91,6 +93,8 @@ class ChatApi(AppApiResource):
         parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
         parser.add_argument('conversation_id', type=uuid_value, location='json')
         parser.add_argument('user', type=str, location='json')
+        parser.add_argument('retriever_from', type=str, required=False, default='dev', location='json')
+
         args = parser.parse_args()
 
         streaming = args['response_mode'] == 'streaming'

api/controllers/service_api/app/message.py

@@ -16,6 +16,24 @@ class MessageListApi(AppApiResource):
     feedback_fields = {
         'rating': fields.String
     }
+    retriever_resource_fields = {
+        'id': fields.String,
+        'message_id': fields.String,
+        'position': fields.Integer,
+        'dataset_id': fields.String,
+        'dataset_name': fields.String,
+        'document_id': fields.String,
+        'document_name': fields.String,
+        'data_source_type': fields.String,
+        'segment_id': fields.String,
+        'score': fields.Float,
+        'hit_count': fields.Integer,
+        'word_count': fields.Integer,
+        'segment_position': fields.Integer,
+        'index_node_hash': fields.String,
+        'content': fields.String,
+        'created_at': TimestampField
+    }
 
     message_fields = {
         'id': fields.String,
@@ -24,6 +42,7 @@ class MessageListApi(AppApiResource):
         'query': fields.String,
         'answer': fields.String,
         'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
+        'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
         'created_at': TimestampField
     }
 

api/controllers/web/app.py

@@ -24,6 +24,7 @@ class AppParameterApi(WebApiResource):
         'suggested_questions': fields.Raw,
         'suggested_questions_after_answer': fields.Raw,
         'speech_to_text': fields.Raw,
+        'retriever_resource': fields.Raw,
         'more_like_this': fields.Raw,
         'user_input_form': fields.Raw,
     }
@@ -38,6 +39,7 @@ class AppParameterApi(WebApiResource):
             'suggested_questions': app_model_config.suggested_questions_list,
             'suggested_questions_after_answer': app_model_config.suggested_questions_after_answer_dict,
             'speech_to_text': app_model_config.speech_to_text_dict,
+            'retriever_resource': app_model_config.retriever_resource_dict,
             'more_like_this': app_model_config.more_like_this_dict,
             'user_input_form': app_model_config.user_input_form_list
         }

api/controllers/web/completion.py

@@ -29,8 +29,10 @@ class CompletionApi(WebApiResource):
 
         parser = reqparse.RequestParser()
         parser.add_argument('inputs', type=dict, required=True, location='json')
-        parser.add_argument('query', type=str, location='json')
+        parser.add_argument('query', type=str, location='json', default='')
         parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
+        parser.add_argument('retriever_from', type=str, required=False, default='web_app', location='json')
+
         args = parser.parse_args()
 
         streaming = args['response_mode'] == 'streaming'
@@ -88,6 +90,8 @@ class ChatApi(WebApiResource):
         parser.add_argument('query', type=str, required=True, location='json')
         parser.add_argument('response_mode', type=str, choices=['blocking', 'streaming'], location='json')
         parser.add_argument('conversation_id', type=uuid_value, location='json')
+        parser.add_argument('retriever_from', type=str, required=False, default='web_app', location='json')
+
         args = parser.parse_args()
 
         streaming = args['response_mode'] == 'streaming'

api/controllers/web/message.py

@@ -29,6 +29,25 @@ class MessageListApi(WebApiResource):
         'rating': fields.String
     }
 
+    retriever_resource_fields = {
+        'id': fields.String,
+        'message_id': fields.String,
+        'position': fields.Integer,
+        'dataset_id': fields.String,
+        'dataset_name': fields.String,
+        'document_id': fields.String,
+        'document_name': fields.String,
+        'data_source_type': fields.String,
+        'segment_id': fields.String,
+        'score': fields.Float,
+        'hit_count': fields.Integer,
+        'word_count': fields.Integer,
+        'segment_position': fields.Integer,
+        'index_node_hash': fields.String,
+        'content': fields.String,
+        'created_at': TimestampField
+    }
+
     message_fields = {
         'id': fields.String,
         'conversation_id': fields.String,
@@ -36,6 +55,7 @@ class MessageListApi(WebApiResource):
         'query': fields.String,
         'answer': fields.String,
         'feedback': fields.Nested(feedback_fields, attribute='user_feedback', allow_null=True),
+        'retriever_resources': fields.List(fields.Nested(retriever_resource_fields)),
         'created_at': TimestampField
     }
 

api/core/agent/agent/multi_dataset_router_agent.py

@@ -1,3 +1,4 @@
+import json
 from typing import Tuple, List, Any, Union, Sequence, Optional, cast
 
 from langchain.agents import OpenAIFunctionsAgent, BaseSingleActionAgent
@@ -53,6 +54,10 @@ class MultiDatasetRouterAgent(OpenAIFunctionsAgent):
             tool = next(iter(self.tools))
             tool = cast(DatasetRetrieverTool, tool)
             rst = tool.run(tool_input={'query': kwargs['input']})
+            # output = ''
+            # rst_json = json.loads(rst)
+            # for item in rst_json:
+            #     output += f'{item["content"]}\n'
             return AgentFinish(return_values={"output": rst}, log=rst)
 
         if intermediate_steps:

api/core/callback_handler/dataset_tool_callback_handler.py

@@ -64,12 +64,9 @@ class DatasetToolCallbackHandler(BaseCallbackHandler):
         llm_prefix: Optional[str] = None,
         **kwargs: Any,
     ) -> None:
-        # kwargs={'name': 'Search'}
-        # llm_prefix='Thought:'
-        # observation_prefix='Observation: '
-        # output='53 years'
         pass
 
+
     def on_tool_error(
         self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
     ) -> None:

api/core/callback_handler/index_tool_callback_handler.py

@@ -2,6 +2,7 @@ from typing import List
 
 from langchain.schema import Document
 
+from core.conversation_message_task import ConversationMessageTask
 from extensions.ext_database import db
 from models.dataset import DocumentSegment
 
@@ -9,8 +10,9 @@ from models.dataset import DocumentSegment
 class DatasetIndexToolCallbackHandler:
     """Callback handler for dataset tool."""
 
-    def __init__(self, dataset_id: str) -> None:
+    def __init__(self, dataset_id: str, conversation_message_task: ConversationMessageTask) -> None:
         self.dataset_id = dataset_id
+        self.conversation_message_task = conversation_message_task
 
     def on_tool_end(self, documents: List[Document]) -> None:
         """Handle tool end."""
@@ -27,3 +29,7 @@ class DatasetIndexToolCallbackHandler:
             )
 
             db.session.commit()
+
+    def return_retriever_resource_info(self, resource: List):
+        """Handle return_retriever_resource_info."""
+        self.conversation_message_task.on_dataset_query_finish(resource)

api/core/completion.py

@@ -1,3 +1,4 @@
+import json
 import logging
 import re
 from typing import Optional, List, Union, Tuple
@@ -19,13 +20,15 @@ from core.orchestrator_rule_parser import OrchestratorRuleParser
 from core.prompt.prompt_builder import PromptBuilder
 from core.prompt.prompt_template import JinjaPromptTemplate
 from core.prompt.prompts import MORE_LIKE_THIS_GENERATE_PROMPT
+from models.dataset import DocumentSegment, Dataset, Document
 from models.model import App, AppModelConfig, Account, Conversation, Message, EndUser
 
 
 class Completion:
     @classmethod
     def generate(cls, task_id: str, app: App, app_model_config: AppModelConfig, query: str, inputs: dict,
-                 user: Union[Account, EndUser], conversation: Optional[Conversation], streaming: bool, is_override: bool = False):
+                 user: Union[Account, EndUser], conversation: Optional[Conversation], streaming: bool,
+                 is_override: bool = False, retriever_from: str = 'dev'):
         """
         errors: ProviderTokenNotInitError
         """
@@ -96,7 +99,6 @@ class Completion:
             should_use_agent = agent_executor.should_use_agent(query)
             if should_use_agent:
                 agent_execute_result = agent_executor.run(query)
-
         # run the final llm
         try:
             cls.run_final_llm(
@@ -118,7 +120,8 @@ class Completion:
             return
 
     @classmethod
-    def run_final_llm(cls, model_instance: BaseLLM, mode: str, app_model_config: AppModelConfig, query: str, inputs: dict,
+    def run_final_llm(cls, model_instance: BaseLLM, mode: str, app_model_config: AppModelConfig, query: str,
+                      inputs: dict,
                       agent_execute_result: Optional[AgentExecuteResult],
                       conversation_message_task: ConversationMessageTask,
                       memory: Optional[ReadOnlyConversationTokenDBBufferSharedMemory]):
@@ -150,7 +153,6 @@ class Completion:
             callbacks=[LLMCallbackHandler(model_instance, conversation_message_task)],
             fake_response=fake_response
         )
-
         return response
 
     @classmethod

api/core/conversation_message_task.py

@@ -1,6 +1,6 @@
 import decimal
 import json
-from typing import Optional, Union
+from typing import Optional, Union, List
 
 from core.callback_handler.entity.agent_loop import AgentLoop
 from core.callback_handler.entity.dataset_query import DatasetQueryObj
@@ -15,7 +15,8 @@ from events.message_event import message_was_created
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
 from models.dataset import DatasetQuery
-from models.model import AppModelConfig, Conversation, Account, Message, EndUser, App, MessageAgentThought, MessageChain
+from models.model import AppModelConfig, Conversation, Account, Message, EndUser, App, MessageAgentThought, \
+    MessageChain, DatasetRetrieverResource
 
 
 class ConversationMessageTask:
@@ -41,6 +42,8 @@ class ConversationMessageTask:
 
         self.message = None
 
+        self.retriever_resource = None
+
         self.model_dict = self.app_model_config.model_dict
         self.provider_name = self.model_dict.get('provider')
         self.model_name = self.model_dict.get('name')
@@ -157,7 +160,8 @@ class ConversationMessageTask:
         self.message.message_tokens = message_tokens
         self.message.message_unit_price = message_unit_price
         self.message.message_price_unit = message_price_unit
-        self.message.answer = PromptBuilder.process_template(llm_message.completion.strip()) if llm_message.completion else ''
+        self.message.answer = PromptBuilder.process_template(
+            llm_message.completion.strip()) if llm_message.completion else ''
         self.message.answer_tokens = answer_tokens
         self.message.answer_unit_price = answer_unit_price
         self.message.answer_price_unit = answer_price_unit
@@ -256,7 +260,36 @@ class ConversationMessageTask:
 
         db.session.add(dataset_query)
 
+    def on_dataset_query_finish(self, resource: List):
+        if resource and len(resource) > 0:
+            for item in resource:
+                dataset_retriever_resource = DatasetRetrieverResource(
+                    message_id=self.message.id,
+                    position=item.get('position'),
+                    dataset_id=item.get('dataset_id'),
+                    dataset_name=item.get('dataset_name'),
+                    document_id=item.get('document_id'),
+                    document_name=item.get('document_name'),
+                    data_source_type=item.get('data_source_type'),
+                    segment_id=item.get('segment_id'),
+                    score=item.get('score') if 'score' in item else None,
+                    hit_count=item.get('hit_count') if 'hit_count' else None,
+                    word_count=item.get('word_count') if 'word_count' in item else None,
+                    segment_position=item.get('segment_position') if 'segment_position' in item else None,
+                    index_node_hash=item.get('index_node_hash') if 'index_node_hash' in item else None,
+                    content=item.get('content'),
+                    retriever_from=item.get('retriever_from'),
+                    created_by=self.user.id
+                )
+                db.session.add(dataset_retriever_resource)
+                db.session.flush()
+            self.retriever_resource = resource
+
+    def message_end(self):
+        self._pub_handler.pub_message_end(self.retriever_resource)
+
     def end(self):
+        self._pub_handler.pub_message_end(self.retriever_resource)
         self._pub_handler.pub_end()
 
 
@@ -350,6 +383,23 @@ class PubHandler:
             self.pub_end()
             raise ConversationTaskStoppedException()
 
+    def pub_message_end(self, retriever_resource: List):
+        content = {
+            'event': 'message_end',
+            'data': {
+                'task_id': self._task_id,
+                'message_id': self._message.id,
+                'mode': self._conversation.mode,
+                'conversation_id': self._conversation.id
+            }
+        }
+        if retriever_resource:
+            content['data']['retriever_resources'] = retriever_resource
+        redis_client.publish(self._channel, json.dumps(content))
+
+        if self._is_stopped():
+            self.pub_end()
+            raise ConversationTaskStoppedException()
 
     def pub_end(self):
         content = {

api/core/generator/llm_generator.py

@@ -1,3 +1,4 @@
+import json
 import logging
 
 from langchain.schema import OutputParserException
@@ -22,18 +23,25 @@ class LLMGenerator:
         if len(query) > 2000:
             query = query[:300] + "...[TRUNCATED]..." + query[-300:]
 
-        prompt = prompt.format(query=query)
+        query = query.replace("\n", " ")
+
+        prompt += query + "\n"
 
         model_instance = ModelFactory.get_text_generation_model(
             tenant_id=tenant_id,
             model_kwargs=ModelKwargs(
-                max_tokens=50
+                temperature=1,
+                max_tokens=100
             )
         )
 
         prompts = [PromptMessage(content=prompt)]
         response = model_instance.run(prompts)
         answer = response.content
+
+        result_dict = json.loads(answer)
+        answer = result_dict['Your Output']
+
         return answer.strip()
 
     @classmethod

api/core/index/keyword_table_index/keyword_table_index.py

@@ -25,7 +25,7 @@ class KeywordTableIndex(BaseIndex):
         keyword_table = {}
         for text in texts:
             keywords = keyword_table_handler.extract_keywords(text.page_content, self._config.max_keywords_per_chunk)
-            self._update_segment_keywords(text.metadata['doc_id'], list(keywords))
+            self._update_segment_keywords(self.dataset.id, text.metadata['doc_id'], list(keywords))
             keyword_table = self._add_text_to_keyword_table(keyword_table, text.metadata['doc_id'], list(keywords))
 
         dataset_keyword_table = DatasetKeywordTable(
@@ -52,7 +52,7 @@ class KeywordTableIndex(BaseIndex):
         keyword_table = self._get_dataset_keyword_table()
         for text in texts:
             keywords = keyword_table_handler.extract_keywords(text.page_content, self._config.max_keywords_per_chunk)
-            self._update_segment_keywords(text.metadata['doc_id'], list(keywords))
+            self._update_segment_keywords(self.dataset.id, text.metadata['doc_id'], list(keywords))
             keyword_table = self._add_text_to_keyword_table(keyword_table, text.metadata['doc_id'], list(keywords))
 
         self._save_dataset_keyword_table(keyword_table)
@@ -74,7 +74,7 @@ class KeywordTableIndex(BaseIndex):
             DocumentSegment.document_id == document_id
         ).all()
 
-        ids = [segment.id for segment in segments]
+        ids = [segment.index_node_id for segment in segments]
 
         keyword_table = self._get_dataset_keyword_table()
         keyword_table = self._delete_ids_from_keyword_table(keyword_table, ids)
@@ -199,15 +199,18 @@ class KeywordTableIndex(BaseIndex):
 
         return sorted_chunk_indices[: k]
 
-    def _update_segment_keywords(self, node_id: str, keywords: List[str]):
-        document_segment = db.session.query(DocumentSegment).filter(DocumentSegment.index_node_id == node_id).first()
+    def _update_segment_keywords(self, dataset_id: str, node_id: str, keywords: List[str]):
+        document_segment = db.session.query(DocumentSegment).filter(
+            DocumentSegment.dataset_id == dataset_id,
+            DocumentSegment.index_node_id == node_id
+        ).first()
         if document_segment:
             document_segment.keywords = keywords
             db.session.commit()
 
     def create_segment_keywords(self, node_id: str, keywords: List[str]):
         keyword_table = self._get_dataset_keyword_table()
-        self._update_segment_keywords(node_id, keywords)
+        self._update_segment_keywords(self.dataset.id, node_id, keywords)
         keyword_table = self._add_text_to_keyword_table(keyword_table, node_id, keywords)
         self._save_dataset_keyword_table(keyword_table)
 

api/core/index/vector_index/qdrant_vector_index.py

@@ -113,6 +113,25 @@ class QdrantVectorIndex(BaseVectorIndex):
             ],
         ))
 
+    def delete_by_ids(self, ids: list[str]) -> None:
+        if self._is_origin():
+            self.recreate_dataset(self.dataset)
+            return
+
+        vector_store = self._get_vector_store()
+        vector_store = cast(self._get_vector_store_class(), vector_store)
+
+        from qdrant_client.http import models
+        for node_id in ids:
+            vector_store.del_texts(models.Filter(
+                must=[
+                    models.FieldCondition(
+                        key="metadata.doc_id",
+                        match=models.MatchValue(value=node_id),
+                    ),
+                ],
+            ))
+
     def _is_origin(self):
         if self.dataset.index_struct_dict:
             class_prefix: str = self.dataset.index_struct_dict['vector_store']['class_prefix']

api/core/model_providers/models/entity/message.py

@@ -8,6 +8,7 @@ class LLMRunResult(BaseModel):
     content: str
     prompt_tokens: int
     completion_tokens: int
+    source: list = None
 
 
 class MessageType(enum.Enum):

api/core/model_providers/models/llm/base.py

@@ -342,7 +342,7 @@ class BaseLLM(BaseProviderModel):
             if order == 'context_prompt':
                 prompt += context_prompt_content
             elif order == 'pre_prompt':
-                prompt += (pre_prompt_content + '\n\n') if pre_prompt_content else ''
+                prompt += pre_prompt_content
 
         query_prompt = prompt_rules['query_prompt'] if 'query_prompt' in prompt_rules else '{{query}}'
 

api/core/model_providers/models/llm/huggingface_hub_model.py

@@ -1,6 +1,5 @@
 from typing import List, Optional, Any
 
-from langchain import HuggingFaceHub
 from langchain.callbacks.manager import Callbacks
 from langchain.schema import LLMResult
 
@@ -9,6 +8,7 @@ from core.model_providers.models.llm.base import BaseLLM
 from core.model_providers.models.entity.message import PromptMessage
 from core.model_providers.models.entity.model_params import ModelMode, ModelKwargs
 from core.third_party.langchain.llms.huggingface_endpoint_llm import HuggingFaceEndpointLLM
+from core.third_party.langchain.llms.huggingface_hub_llm import HuggingFaceHubLLM
 
 
 class HuggingfaceHubModel(BaseLLM):
@@ -31,7 +31,7 @@ class HuggingfaceHubModel(BaseLLM):
                 streaming=streaming
             )
         else:
-            client = HuggingFaceHub(
+            client = HuggingFaceHubLLM(
                 repo_id=self.name,
                 task=self.credentials['task_type'],
                 model_kwargs=provider_model_kwargs,
@@ -88,4 +88,6 @@ class HuggingfaceHubModel(BaseLLM):
             if 'baichuan' in self.name.lower():
                 return False
 
-        return True
+            return True
+        else:
+            return False

api/core/model_providers/providers/huggingface_hub_provider.py

@@ -89,7 +89,8 @@ class HuggingfaceHubProvider(BaseModelProvider):
                 raise CredentialsValidateFailedError('Task Type must be provided.')
 
             if credentials['task_type'] not in ("text2text-generation", "text-generation", "summarization"):
-                raise CredentialsValidateFailedError('Task Type must be one of text2text-generation, text-generation, summarization.')
+                raise CredentialsValidateFailedError('Task Type must be one of text2text-generation, '
+                                                     'text-generation, summarization.')
 
             try:
                 llm = HuggingFaceEndpointLLM(

api/core/model_providers/providers/spark_provider.py

@@ -83,14 +83,15 @@ class SparkProvider(BaseModelProvider):
         if 'api_secret' not in credentials:
             raise CredentialsValidateFailedError('Spark api_secret must be provided.')
 
-        try:
-            credential_kwargs = {
-                'app_id': credentials['app_id'],
-                'api_key': credentials['api_key'],
-                'api_secret': credentials['api_secret'],
-            }
+        credential_kwargs = {
+            'app_id': credentials['app_id'],
+            'api_key': credentials['api_key'],
+            'api_secret': credentials['api_secret'],
+        }
 
+        try:
             chat_llm = ChatSpark(
+                model_name='spark-v2',
                 max_tokens=10,
                 temperature=0.01,
                 **credential_kwargs
@@ -104,7 +105,27 @@ class SparkProvider(BaseModelProvider):
 
             chat_llm(messages)
         except SparkError as ex:
-            raise CredentialsValidateFailedError(str(ex))
+            # try spark v1.5 if v2.1 failed
+            try:
+                chat_llm = ChatSpark(
+                    model_name='spark',
+                    max_tokens=10,
+                    temperature=0.01,
+                    **credential_kwargs
+                )
+
+                messages = [
+                    HumanMessage(
+                        content="ping"
+                    )
+                ]
+
+                chat_llm(messages)
+            except SparkError as ex:
+                raise CredentialsValidateFailedError(str(ex))
+            except Exception as ex:
+                logging.exception('Spark config validation failed')
+                raise ex
         except Exception as ex:
             logging.exception('Spark config validation failed')
             raise ex

api/core/orchestrator_rule_parser.py

@@ -36,8 +36,8 @@ class OrchestratorRuleParser:
         self.app_model_config = app_model_config
 
     def to_agent_executor(self, conversation_message_task: ConversationMessageTask, memory: Optional[BaseChatMemory],
-                          rest_tokens: int, chain_callback: MainChainGatherCallbackHandler) \
-            -> Optional[AgentExecutor]:
+                          rest_tokens: int, chain_callback: MainChainGatherCallbackHandler,
+                          return_resource: bool = False, retriever_from: str = 'dev') -> Optional[AgentExecutor]:
         if not self.app_model_config.agent_mode_dict:
             return None
 
@@ -74,7 +74,7 @@ class OrchestratorRuleParser:
 
             # only OpenAI chat model (include Azure) support function call, use ReACT instead
             if agent_model_instance.model_mode != ModelMode.CHAT \
-                         or agent_model_instance.model_provider.provider_name not in ['openai', 'azure_openai']:
+                    or agent_model_instance.model_provider.provider_name not in ['openai', 'azure_openai']:
                 if planning_strategy in [PlanningStrategy.FUNCTION_CALL, PlanningStrategy.MULTI_FUNCTION_CALL]:
                     planning_strategy = PlanningStrategy.REACT
                 elif planning_strategy == PlanningStrategy.ROUTER:
@@ -99,7 +99,9 @@ class OrchestratorRuleParser:
                 tool_configs=tool_configs,
                 conversation_message_task=conversation_message_task,
                 rest_tokens=rest_tokens,
-                callbacks=[agent_callback, DifyStdOutCallbackHandler()]
+                callbacks=[agent_callback, DifyStdOutCallbackHandler()],
+                return_resource=return_resource,
+                retriever_from=retriever_from
             )
 
             if len(tools) == 0:
@@ -145,8 +147,10 @@ class OrchestratorRuleParser:
 
         return None
 
-    def to_tools(self, agent_model_instance: BaseLLM, tool_configs: list, conversation_message_task: ConversationMessageTask,
-                 rest_tokens: int, callbacks: Callbacks = None) -> list[BaseTool]:
+    def to_tools(self, agent_model_instance: BaseLLM, tool_configs: list,
+                 conversation_message_task: ConversationMessageTask,
+                 rest_tokens: int, callbacks: Callbacks = None, return_resource: bool = False,
+                 retriever_from: str = 'dev') -> list[BaseTool]:
         """
         Convert app agent tool configs to tools
 
@@ -155,6 +159,8 @@ class OrchestratorRuleParser:
         :param tool_configs: app agent tool configs
         :param conversation_message_task:
         :param callbacks:
+        :param return_resource:
+        :param retriever_from:
         :return:
         """
         tools = []
@@ -166,7 +172,7 @@ class OrchestratorRuleParser:
 
             tool = None
             if tool_type == "dataset":
-                tool = self.to_dataset_retriever_tool(tool_val, conversation_message_task, rest_tokens)
+                tool = self.to_dataset_retriever_tool(tool_val, conversation_message_task, rest_tokens, return_resource, retriever_from)
             elif tool_type == "web_reader":
                 tool = self.to_web_reader_tool(agent_model_instance)
             elif tool_type == "google_search":
@@ -183,13 +189,15 @@ class OrchestratorRuleParser:
         return tools
 
     def to_dataset_retriever_tool(self, tool_config: dict, conversation_message_task: ConversationMessageTask,
-                                  rest_tokens: int) \
+                                  rest_tokens: int, return_resource: bool = False, retriever_from: str = 'dev') \
             -> Optional[BaseTool]:
         """
         A dataset tool is a tool that can be used to retrieve information from a dataset
         :param rest_tokens:
         :param tool_config:
         :param conversation_message_task:
+        :param return_resource:
+        :param retriever_from:
         :return:
         """
         # get dataset from dataset id
@@ -208,7 +216,10 @@ class OrchestratorRuleParser:
         tool = DatasetRetrieverTool.from_dataset(
             dataset=dataset,
             k=k,
-            callbacks=[DatasetToolCallbackHandler(conversation_message_task)]
+            callbacks=[DatasetToolCallbackHandler(conversation_message_task)],
+            conversation_message_task=conversation_message_task,
+            return_resource=return_resource,
+            retriever_from=retriever_from
         )
 
         return tool

api/core/prompt/generate_prompts/baichuan_chat.json

@@ -8,6 +8,6 @@
     "pre_prompt",
     "histories_prompt"
   ],
-  "query_prompt": "用户：{{query}}",
+  "query_prompt": "\n\n用户：{{query}}",
   "stops": ["用户："]
 }

api/core/prompt/generate_prompts/common_chat.json

@@ -8,6 +8,6 @@
     "pre_prompt",
     "histories_prompt"
   ],
-  "query_prompt": "Human: {{query}}\n\nAssistant: ",
+  "query_prompt": "\n\nHuman: {{query}}\n\nAssistant: ",
   "stops": ["\nHuman:", "</histories>"]
-}
+}

api/core/prompt/prompts.py

@@ -1,10 +1,65 @@
-CONVERSATION_TITLE_PROMPT = (
-    "Human:{query}\n-----\n"
-    "Help me summarize the intent of what the human said and provide a title, the title should not exceed 20 words.\n"
-    "If what the human said is conducted in English, you should only return an English title.\n" 
-    "If what the human said is conducted in Chinese, you should only return a Chinese title.\n"
-    "title:"
-)
+# Written by YORKI MINAKO🤡
+CONVERSATION_TITLE_PROMPT = """You need to decompose the user's input into "subject" and "intention" in order to accurately figure out what the user's input language actually is. 
+Notice: the language type user use could be diverse, which can be English, Chinese, Español, Arabic, Japanese, French, and etc.
+MAKE SURE your output is the SAME language as the user's input!
+Your output is restricted only to: (Input language) Intention + Subject(short as possible)
+Your output MUST be a valid JSON.
+
+Tip: When the user's question is directed at you (the language model), you can add an emoji to make it more fun.
+
+
+example 1:
+User Input: hi, yesterday i had some burgers.
+{
+  "Language Type": "The user's input is pure English",
+  "Your Reasoning": "The language of my output must be pure English.",
+  "Your Output": "sharing yesterday's food"
+}
+
+example 2:
+User Input: hello
+{
+  "Language Type": "The user's input is written in pure English",
+  "Your Reasoning": "The language of my output must be pure English.",
+  "Your Output": "Greeting myself☺️"
+}
+
+
+example 3:
+User Input: why mmap file: oom
+{
+  "Language Type": "The user's input is written in pure English",
+  "Your Reasoning": "The language of my output must be pure English.",
+  "Your Output": "Asking about the reason for mmap file: oom"
+}
+
+
+example 4:
+User Input: www.convinceme.yesterday-you-ate-seafood.tv讲了什么？
+{
+  "Language Type": "The user's input English-Chinese mixed",
+  "Your Reasoning": "The English-part is an URL, the main intention is still written in Chinese, so the language of my output must be using Chinese.",
+  "Your Output": "询问网站www.convinceme.yesterday-you-ate-seafood.tv"
+}
+
+example 5:
+User Input: why小红的年龄is老than小明？
+{
+  "Language Type": "The user's input is English-Chinese mixed",
+  "Your Reasoning": "The English parts are subjective particles, the main intention is written in Chinese, besides, Chinese occupies a greater \"actual meaning\" than English, so the language of my output must be using Chinese.",
+  "Your Output": "询问小红和小明的年龄"
+}
+
+example 6:
+User Input: yo, 你今天咋样？
+{
+  "Language Type": "The user's input is English-Chinese mixed",
+  "Your Reasoning": "The English-part is a subjective particle, the main intention is written in Chinese, so the language of my output must be using Chinese.",
+  "Your Output": "查询今日我的状态☺️"
+}
+
+User Input: 
+"""
 
 CONVERSATION_SUMMARY_PROMPT = (
     "Please generate a short summary of the following conversation.\n"
@@ -50,7 +105,7 @@ GENERATOR_QA_PROMPT = (
     'Step 3: Decompose or combine multiple pieces of information and concepts.\n'
     'Step 4: Generate 20 questions and answers based on these key information and concepts.'
     'The questions should be clear and detailed, and the answers should be detailed and complete.\n'
-    "Answer must be the language:{language} and in the following format: Q1:\nA1:\nQ2:\nA2:...\n"
+    "Answer according to the the language:{language} and in the following format: Q1:\nA1:\nQ2:\nA2:...\n"
 )
 
 RULE_CONFIG_GENERATE_TEMPLATE = """Given MY INTENDED AUDIENCES and HOPING TO SOLVE using a language model, please select \

api/core/third_party/langchain/llms/huggingface_hub_llm.py

@@ -0,0 +1,62 @@
+from typing import Dict, Optional, List, Any
+
+from huggingface_hub import HfApi, InferenceApi
+from langchain import HuggingFaceHub
+from langchain.callbacks.manager import CallbackManagerForLLMRun
+from langchain.llms.huggingface_hub import VALID_TASKS
+from pydantic import root_validator
+
+from langchain.utils import get_from_dict_or_env
+
+
+class HuggingFaceHubLLM(HuggingFaceHub):
+    """HuggingFaceHub  models.
+
+    To use, you should have the ``huggingface_hub`` python package installed, and the
+    environment variable ``HUGGINGFACEHUB_API_TOKEN`` set with your API token, or pass
+    it as a named parameter to the constructor.
+
+    Only supports `text-generation`, `text2text-generation` and `summarization` for now.
+
+    Example:
+        .. code-block:: python
+
+            from langchain.llms import HuggingFaceHub
+            hf = HuggingFaceHub(repo_id="gpt2", huggingfacehub_api_token="my-api-key")
+    """
+
+    @root_validator()
+    def validate_environment(cls, values: Dict) -> Dict:
+        """Validate that api key and python package exists in environment."""
+        huggingfacehub_api_token = get_from_dict_or_env(
+            values, "huggingfacehub_api_token", "HUGGINGFACEHUB_API_TOKEN"
+        )
+        client = InferenceApi(
+            repo_id=values["repo_id"],
+            token=huggingfacehub_api_token,
+            task=values.get("task"),
+        )
+        client.options = {"wait_for_model": False, "use_gpu": False}
+        values["client"] = client
+        return values
+
+    def _call(
+            self,
+            prompt: str,
+            stop: Optional[List[str]] = None,
+            run_manager: Optional[CallbackManagerForLLMRun] = None,
+            **kwargs: Any,
+    ) -> str:
+        hfapi = HfApi(token=self.huggingfacehub_api_token)
+        model_info = hfapi.model_info(repo_id=self.repo_id)
+        if not model_info:
+            raise ValueError(f"Model {self.repo_id} not found.")
+
+        if 'inference' in model_info.cardData and not model_info.cardData['inference']:
+            raise ValueError(f"Inference API has been turned off for this model {self.repo_id}.")
+
+        if model_info.pipeline_tag not in VALID_TASKS:
+            raise ValueError(f"Model {self.repo_id} is not a valid task, "
+                             f"must be one of {VALID_TASKS}.")
+
+        return super()._call(prompt, stop, run_manager, **kwargs)

api/core/tool/dataset_retriever_tool.py

@@ -1,3 +1,4 @@
+import json
 from typing import Type
 
 from flask import current_app
@@ -5,13 +6,14 @@ from langchain.tools import BaseTool
 from pydantic import Field, BaseModel
 
 from core.callback_handler.index_tool_callback_handler import DatasetIndexToolCallbackHandler
+from core.conversation_message_task import ConversationMessageTask
 from core.embedding.cached_embedding import CacheEmbedding
 from core.index.keyword_table_index.keyword_table_index import KeywordTableIndex, KeywordTableConfig
 from core.index.vector_index.vector_index import VectorIndex
 from core.model_providers.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.model_providers.model_factory import ModelFactory
 from extensions.ext_database import db
-from models.dataset import Dataset, DocumentSegment
+from models.dataset import Dataset, DocumentSegment, Document
 
 
 class DatasetRetrieverToolInput(BaseModel):
@@ -27,6 +29,10 @@ class DatasetRetrieverTool(BaseTool):
     tenant_id: str
     dataset_id: str
     k: int = 3
+    conversation_message_task: ConversationMessageTask
+    return_resource: str
+    retriever_from: str
+
 
     @classmethod
     def from_dataset(cls, dataset: Dataset, **kwargs):
@@ -86,7 +92,7 @@ class DatasetRetrieverTool(BaseTool):
             if self.k > 0:
                 documents = vector_index.search(
                     query,
-                    search_type='similarity',
+                    search_type='similarity_score_threshold',
                     search_kwargs={
                         'k': self.k
                     }
@@ -94,8 +100,12 @@ class DatasetRetrieverTool(BaseTool):
             else:
                 documents = []
 
-            hit_callback = DatasetIndexToolCallbackHandler(dataset.id)
+            hit_callback = DatasetIndexToolCallbackHandler(dataset.id, self.conversation_message_task)
             hit_callback.on_tool_end(documents)
+            document_score_list = {}
+            if dataset.indexing_technique != "economy":
+                for item in documents:
+                    document_score_list[item.metadata['doc_id']] = item.metadata['score']
             document_context_list = []
             index_node_ids = [document.metadata['doc_id'] for document in documents]
             segments = DocumentSegment.query.filter(DocumentSegment.dataset_id == self.dataset_id,
@@ -112,9 +122,43 @@ class DatasetRetrieverTool(BaseTool):
                                                                                            float('inf')))
                 for segment in sorted_segments:
                     if segment.answer:
-                        document_context_list.append(f'question:{segment.content} \nanswer:{segment.answer}')
+                        document_context_list.append(f'question:{segment.content} answer:{segment.answer}')
                     else:
                         document_context_list.append(segment.content)
+                if self.return_resource:
+                    context_list = []
+                    resource_number = 1
+                    for segment in sorted_segments:
+                        context = {}
+                        document = Document.query.filter(Document.id == segment.document_id,
+                                                         Document.enabled == True,
+                                                         Document.archived == False,
+                                                         ).first()
+                        if dataset and document:
+                            source = {
+                                'position': resource_number,
+                                'dataset_id': dataset.id,
+                                'dataset_name': dataset.name,
+                                'document_id': document.id,
+                                'document_name': document.name,
+                                'data_source_type': document.data_source_type,
+                                'segment_id': segment.id,
+                                'retriever_from': self.retriever_from
+                            }
+                            if dataset.indexing_technique != "economy":
+                                source['score'] = document_score_list.get(segment.index_node_id)
+                            if self.retriever_from == 'dev':
+                                source['hit_count'] = segment.hit_count
+                                source['word_count'] = segment.word_count
+                                source['segment_position'] = segment.position
+                                source['index_node_hash'] = segment.index_node_hash
+                            if segment.answer:
+                                source['content'] = f'question:{segment.content} \nanswer:{segment.answer}'
+                            else:
+                                source['content'] = segment.content
+                            context_list.append(source)
+                        resource_number += 1
+                    hit_callback.return_retriever_resource_info(context_list)
 
             return str("\n".join(document_context_list))
 

api/events/event_handlers/generate_conversation_name_when_first_message_created.py

@@ -26,7 +26,7 @@ def handle(sender, **kwargs):
 
                 conversation.name = name
             except:
-                conversation.name = 'New Chat'
+                conversation.name = 'New conversation'
 
             db.session.add(conversation)
             db.session.commit()

api/migrations/versions/6dcb43972bdc_add_dataset_retriever_resource.py

@@ -0,0 +1,54 @@
+"""add_dataset_retriever_resource
+
+Revision ID: 6dcb43972bdc
+Revises: 4bcffcd64aa4
+Create Date: 2023-09-06 16:51:27.385844
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = '6dcb43972bdc'
+down_revision = '4bcffcd64aa4'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('dataset_retriever_resources',
+    sa.Column('id', postgresql.UUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+    sa.Column('message_id', postgresql.UUID(), nullable=False),
+    sa.Column('position', sa.Integer(), nullable=False),
+    sa.Column('dataset_id', postgresql.UUID(), nullable=False),
+    sa.Column('dataset_name', sa.Text(), nullable=False),
+    sa.Column('document_id', postgresql.UUID(), nullable=False),
+    sa.Column('document_name', sa.Text(), nullable=False),
+    sa.Column('data_source_type', sa.Text(), nullable=False),
+    sa.Column('segment_id', postgresql.UUID(), nullable=False),
+    sa.Column('score', sa.Float(), nullable=True),
+    sa.Column('content', sa.Text(), nullable=False),
+    sa.Column('hit_count', sa.Integer(), nullable=True),
+    sa.Column('word_count', sa.Integer(), nullable=True),
+    sa.Column('segment_position', sa.Integer(), nullable=True),
+    sa.Column('index_node_hash', sa.Text(), nullable=True),
+    sa.Column('retriever_from', sa.Text(), nullable=False),
+    sa.Column('created_by', postgresql.UUID(), nullable=False),
+    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+    sa.PrimaryKeyConstraint('id', name='dataset_retriever_resource_pkey')
+    )
+    with op.batch_alter_table('dataset_retriever_resources', schema=None) as batch_op:
+        batch_op.create_index('dataset_retriever_resource_message_id_idx', ['message_id'], unique=False)
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('dataset_retriever_resources', schema=None) as batch_op:
+        batch_op.drop_index('dataset_retriever_resource_message_id_idx')
+
+    op.drop_table('dataset_retriever_resources')
+    # ### end Alembic commands ###

api/migrations/versions/77e83833755c_add_app_config_retriever_resource.py

@@ -0,0 +1,32 @@
+"""add_app_config_retriever_resource
+
+Revision ID: 77e83833755c
+Revises: 6dcb43972bdc
+Create Date: 2023-09-06 17:26:40.311927
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '77e83833755c'
+down_revision = '6dcb43972bdc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('app_model_configs', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('retriever_resource', sa.Text(), nullable=True))
+
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    with op.batch_alter_table('app_model_configs', schema=None) as batch_op:
+        batch_op.drop_column('retriever_resource')
+
+    # ### end Alembic commands ###

api/models/model.py

@@ -1,4 +1,5 @@
 import json
+from json import JSONDecodeError
 
 from flask import current_app, request
 from flask_login import UserMixin
@@ -90,6 +91,7 @@ class AppModelConfig(db.Model):
     pre_prompt = db.Column(db.Text)
     agent_mode = db.Column(db.Text)
     sensitive_word_avoidance = db.Column(db.Text)
+    retriever_resource = db.Column(db.Text)
 
     @property
     def app(self):
@@ -114,6 +116,11 @@ class AppModelConfig(db.Model):
         return json.loads(self.speech_to_text) if self.speech_to_text \
             else {"enabled": False}
 
+    @property
+    def retriever_resource_dict(self) -> dict:
+        return json.loads(self.retriever_resource) if self.retriever_resource \
+            else {"enabled": False}
+
     @property
     def more_like_this_dict(self) -> dict:
         return json.loads(self.more_like_this) if self.more_like_this else {"enabled": False}
@@ -140,6 +147,7 @@ class AppModelConfig(db.Model):
             "suggested_questions": self.suggested_questions_list,
             "suggested_questions_after_answer": self.suggested_questions_after_answer_dict,
             "speech_to_text": self.speech_to_text_dict,
+            "retriever_resource": self.retriever_resource,
             "more_like_this": self.more_like_this_dict,
             "sensitive_word_avoidance": self.sensitive_word_avoidance_dict,
             "model": self.model_dict,
@@ -164,7 +172,8 @@ class AppModelConfig(db.Model):
         self.user_input_form = json.dumps(model_config['user_input_form'])
         self.pre_prompt = model_config['pre_prompt']
         self.agent_mode = json.dumps(model_config['agent_mode'])
-
+        self.retriever_resource = json.dumps(model_config['retriever_resource']) \
+            if model_config.get('retriever_resource') else None
         return self
 
     def copy(self):
@@ -318,6 +327,7 @@ class Conversation(db.Model):
             model_config['suggested_questions'] = app_model_config.suggested_questions_list
             model_config['suggested_questions_after_answer'] = app_model_config.suggested_questions_after_answer_dict
             model_config['speech_to_text'] = app_model_config.speech_to_text_dict
+            model_config['retriever_resource'] = app_model_config.retriever_resource_dict
             model_config['more_like_this'] = app_model_config.more_like_this_dict
             model_config['sensitive_word_avoidance'] = app_model_config.sensitive_word_avoidance_dict
             model_config['user_input_form'] = app_model_config.user_input_form_list
@@ -476,6 +486,11 @@ class Message(db.Model):
         return db.session.query(MessageAgentThought).filter(MessageAgentThought.message_id == self.id) \
             .order_by(MessageAgentThought.position.asc()).all()
 
+    @property
+    def retriever_resources(self):
+        return db.session.query(DatasetRetrieverResource).filter(DatasetRetrieverResource.message_id == self.id) \
+            .order_by(DatasetRetrieverResource.position.asc()).all()
+
 
 class MessageFeedback(db.Model):
     __tablename__ = 'message_feedbacks'
@@ -719,3 +734,31 @@ class MessageAgentThought(db.Model):
     created_by_role = db.Column(db.String, nullable=False)
     created_by = db.Column(UUID, nullable=False)
     created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())
+
+
+class DatasetRetrieverResource(db.Model):
+    __tablename__ = 'dataset_retriever_resources'
+    __table_args__ = (
+        db.PrimaryKeyConstraint('id', name='dataset_retriever_resource_pkey'),
+        db.Index('dataset_retriever_resource_message_id_idx', 'message_id'),
+    )
+
+    id = db.Column(UUID, nullable=False, server_default=db.text('uuid_generate_v4()'))
+    message_id = db.Column(UUID, nullable=False)
+    position = db.Column(db.Integer, nullable=False)
+    dataset_id = db.Column(UUID, nullable=False)
+    dataset_name = db.Column(db.Text, nullable=False)
+    document_id = db.Column(UUID, nullable=False)
+    document_name = db.Column(db.Text, nullable=False)
+    data_source_type = db.Column(db.Text, nullable=False)
+    segment_id = db.Column(UUID, nullable=False)
+    score = db.Column(db.Float, nullable=True)
+    content = db.Column(db.Text, nullable=False)
+    hit_count = db.Column(db.Integer, nullable=True)
+    word_count = db.Column(db.Integer, nullable=True)
+    segment_position = db.Column(db.Integer, nullable=True)
+    index_node_hash = db.Column(db.Text, nullable=True)
+    retriever_from = db.Column(db.Text, nullable=False)
+    created_by = db.Column(UUID, nullable=False)
+    created_at = db.Column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())
+

api/services/account_service.py

@@ -1,5 +1,6 @@
 # -*- coding:utf-8 -*-
 import base64
+import json
 import logging
 import secrets
 import uuid
@@ -346,6 +347,10 @@ class TenantService:
 
 class RegisterService:
 
+    @classmethod
+    def _get_invitation_token_key(cls, token: str) -> str:
+        return f'member_invite:token:{token}'
+
     @classmethod
     def register(cls, email, name, password: str = None, open_id: str = None, provider: str = None) -> Account:
         db.session.begin_nested()
@@ -401,7 +406,7 @@ class RegisterService:
         # send email
         send_invite_member_mail_task.delay(
             to=email,
-            token=cls.generate_invite_token(tenant, account),
+            token=token,
             inviter_name=inviter.name if inviter else 'Dify',
             workspace_id=tenant.id,
             workspace_name=tenant.name,
@@ -412,21 +417,35 @@ class RegisterService:
     @classmethod
     def generate_invite_token(cls, tenant: Tenant, account: Account) -> str:
         token = str(uuid.uuid4())
-        email_hash = sha256(account.email.encode()).hexdigest()
-        cache_key = 'member_invite_token:{}, {}:{}'.format(str(tenant.id), email_hash, token)
-        redis_client.setex(cache_key, 3600, str(account.id))
+        invitation_data = {
+            'account_id': account.id,
+            'email': account.email,
+            'workspace_id': tenant.id,
+        }
+        redis_client.setex(
+            cls._get_invitation_token_key(token),
+            3600,
+            json.dumps(invitation_data)
+        )
         return token
 
     @classmethod
     def revoke_token(cls, workspace_id: str, email: str, token: str):
-        email_hash = sha256(email.encode()).hexdigest()
-        cache_key = 'member_invite_token:{}, {}:{}'.format(workspace_id, email_hash, token)
-        redis_client.delete(cache_key)
+        if workspace_id and email:
+            email_hash = sha256(email.encode()).hexdigest()
+            cache_key = 'member_invite_token:{}, {}:{}'.format(workspace_id, email_hash, token)
+            redis_client.delete(cache_key)
+        else:
+            redis_client.delete(cls._get_invitation_token_key(token))
 
     @classmethod
-    def get_account_if_token_valid(cls, workspace_id: str, email: str, token: str) -> Optional[Account]:
+    def get_invitation_if_token_valid(cls, workspace_id: str, email: str, token: str) -> Optional[Account]:
+        invitation_data = cls._get_invitation_by_token(token, workspace_id, email)
+        if not invitation_data:
+            return None
+
         tenant = db.session.query(Tenant).filter(
-            Tenant.id == workspace_id,
+            Tenant.id == invitation_data['workspace_id'],
             Tenant.status == 'normal'
         ).first()
 
@@ -435,30 +454,43 @@ class RegisterService:
 
         tenant_account = db.session.query(Account, TenantAccountJoin.role).join(
             TenantAccountJoin, Account.id == TenantAccountJoin.account_id
-        ).filter(Account.email == email, TenantAccountJoin.tenant_id == tenant.id).first()
+        ).filter(Account.email == invitation_data['email'], TenantAccountJoin.tenant_id == tenant.id).first()
 
         if not tenant_account:
             return None
 
-        account_id = cls._get_account_id_by_invite_token(workspace_id, email, token)
-        if not account_id:
-            return None
-
         account = tenant_account[0]
         if not account:
             return None
 
-        if account_id != str(account.id):
+        if invitation_data['account_id'] != str(account.id):
             return None
 
-        return account
+        return {
+                'account': account,
+                'data': invitation_data,
+                'tenant': tenant,
+                }
 
     @classmethod
-    def _get_account_id_by_invite_token(cls, workspace_id: str, email: str, token: str) -> Optional[str]:
-        email_hash = sha256(email.encode()).hexdigest()
-        cache_key = 'member_invite_token:{}, {}:{}'.format(workspace_id, email_hash, token)
-        account_id = redis_client.get(cache_key)
-        if not account_id:
-            return None
+    def _get_invitation_by_token(cls, token: str, workspace_id: str, email: str) -> Optional[str]:
+        if workspace_id is not None and email is not None:
+            email_hash = sha256(email.encode()).hexdigest()
+            cache_key = f'member_invite_token:{workspace_id}, {email_hash}:{token}'
+            account_id = redis_client.get(cache_key)
 
-        return account_id.decode('utf-8')
+            if not account_id:
+                return None
+
+            return {
+                'account_id': account_id.decode('utf-8'),
+                'email': email,
+                'workspace_id': workspace_id,
+            }
+        else:
+            data = redis_client.get(cls._get_invitation_token_key(token))
+            if not data:
+                return None
+
+            invitation = json.loads(data)
+            return invitation

api/services/app_model_config_service.py

@@ -130,6 +130,21 @@ class AppModelConfigService:
         if not isinstance(config["speech_to_text"]["enabled"], bool):
             raise ValueError("enabled in speech_to_text must be of boolean type")
 
+        # return retriever resource
+        if 'retriever_resource' not in config or not config["retriever_resource"]:
+            config["retriever_resource"] = {
+                "enabled": False
+            }
+
+        if not isinstance(config["retriever_resource"], dict):
+            raise ValueError("retriever_resource must be of dict type")
+
+        if "enabled" not in config["retriever_resource"] or not config["retriever_resource"]["enabled"]:
+            config["retriever_resource"]["enabled"] = False
+
+        if not isinstance(config["retriever_resource"]["enabled"], bool):
+            raise ValueError("enabled in speech_to_text must be of boolean type")
+
         # more_like_this
         if 'more_like_this' not in config or not config["more_like_this"]:
             config["more_like_this"] = {
@@ -216,8 +231,8 @@ class AppModelConfigService:
         variables = []
         for item in config["user_input_form"]:
             key = list(item.keys())[0]
-            if key not in ["text-input", "select"]:
-                raise ValueError("Keys in user_input_form list can only be 'text-input' or 'select'")
+            if key not in ["text-input", "select", "paragraph"]:
+                raise ValueError("Keys in user_input_form list can only be 'text-input', 'paragraph'  or 'select'")
 
             form_item = item[key]
             if 'label' not in form_item:
@@ -327,6 +342,7 @@ class AppModelConfigService:
             "suggested_questions": config["suggested_questions"],
             "suggested_questions_after_answer": config["suggested_questions_after_answer"],
             "speech_to_text": config["speech_to_text"],
+            "retriever_resource": config["retriever_resource"],
             "more_like_this": config["more_like_this"],
             "sensitive_word_avoidance": config["sensitive_word_avoidance"],
             "model": {

api/services/completion_service.py

@@ -11,7 +11,8 @@ from sqlalchemy import and_
 
 from core.completion import Completion
 from core.conversation_message_task import PubHandler, ConversationTaskStoppedException
-from core.model_providers.error import LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError, LLMRateLimitError, \
+from core.model_providers.error import LLMBadRequestError, LLMAPIConnectionError, LLMAPIUnavailableError, \
+    LLMRateLimitError, \
     LLMAuthorizationError, ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
@@ -34,7 +35,7 @@ class CompletionService:
         inputs = args['inputs']
         query = args['query']
 
-        if not query:
+        if app_model.mode != 'completion' and not query:
             raise ValueError('query is required')
 
         query = query.replace('\x00', '')
@@ -95,6 +96,7 @@ class CompletionService:
 
                 app_model_config_model = app_model_config.model_dict
                 app_model_config_model['completion_params'] = completion_params
+                app_model_config.retriever_resource = json.dumps({'enabled': True})
 
                 app_model_config = app_model_config.copy()
                 app_model_config.model = json.dumps(app_model_config_model)
@@ -145,7 +147,8 @@ class CompletionService:
             'user': user,
             'conversation': conversation,
             'streaming': streaming,
-            'is_model_config_override': is_model_config_override
+            'is_model_config_override': is_model_config_override,
+            'retriever_from': args['retriever_from'] if 'retriever_from' in args else 'dev'
         })
 
         generate_worker_thread.start()
@@ -169,7 +172,8 @@ class CompletionService:
     @classmethod
     def generate_worker(cls, flask_app: Flask, generate_task_id: str, app_model: App, app_model_config: AppModelConfig,
                         query: str, inputs: dict, user: Union[Account, EndUser],
-                        conversation: Conversation, streaming: bool, is_model_config_override: bool):
+                        conversation: Conversation, streaming: bool, is_model_config_override: bool,
+                        retriever_from: str = 'dev'):
         with flask_app.app_context():
             try:
                 if conversation:
@@ -188,6 +192,7 @@ class CompletionService:
                     conversation=conversation,
                     streaming=streaming,
                     is_override=is_model_config_override,
+                    retriever_from=retriever_from
                 )
             except ConversationTaskStoppedException:
                 pass
@@ -347,8 +352,8 @@ class CompletionService:
                 if value not in options:
                     raise ValueError(f"{variable} in input form must be one of the following: {options}")
             else:
-                if 'max_length' in variable:
-                    max_length = variable['max_length']
+                if 'max_length' in input_config:
+                    max_length = input_config['max_length']
                     if len(value) > max_length:
                         raise ValueError(f'{variable} in input form must be less than {max_length} characters')
 
@@ -367,7 +372,7 @@ class CompletionService:
                         result = json.loads(result)
                         if result.get('error'):
                             cls.handle_error(result)
-                        if 'data' in result:
+                        if result['event'] == 'message' and 'data' in result:
                             return cls.get_message_response_data(result.get('data'))
             except ValueError as e:
                 if e.args[0] != "I/O operation on closed file.":  # ignore this error
@@ -400,7 +405,11 @@ class CompletionService:
                             elif event == 'chain':
                                 yield "data: " + json.dumps(cls.get_chain_response_data(result.get('data'))) + "\n\n"
                             elif event == 'agent_thought':
-                                yield "data: " + json.dumps(cls.get_agent_thought_response_data(result.get('data'))) + "\n\n"
+                                yield "data: " + json.dumps(
+                                    cls.get_agent_thought_response_data(result.get('data'))) + "\n\n"
+                            elif event == 'message_end':
+                                yield "data: " + json.dumps(
+                                    cls.get_message_end_data(result.get('data'))) + "\n\n"
                             elif event == 'ping':
                                 yield "event: ping\n\n"
                             else:
@@ -432,6 +441,20 @@ class CompletionService:
 
         return response_data
 
+    @classmethod
+    def get_message_end_data(cls, data: dict):
+        response_data = {
+            'event': 'message_end',
+            'task_id': data.get('task_id'),
+            'id': data.get('message_id')
+        }
+        if 'retriever_resources' in data:
+            response_data['retriever_resources'] = data.get('retriever_resources')
+        if data.get('mode') == 'chat':
+            response_data['conversation_id'] = data.get('conversation_id')
+
+        return response_data
+
     @classmethod
     def get_chain_response_data(cls, data: dict):
         response_data = {

docker/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 services:
   # API service
   api:
-    image: langgenius/dify-api:0.3.19
+    image: langgenius/dify-api:0.3.22
     restart: always
     environment:
       # Startup mode, 'api' starts the API server.
@@ -124,7 +124,7 @@ services:
   # worker service
   # The Celery worker for processing the queue.
   worker:
-    image: langgenius/dify-api:0.3.19
+    image: langgenius/dify-api:0.3.22
     restart: always
     environment:
       # Startup mode, 'worker' starts the Celery worker for processing the queue.
@@ -176,7 +176,7 @@ services:
 
   # Frontend web application.
   web:
-    image: langgenius/dify-web:0.3.19
+    image: langgenius/dify-web:0.3.22
     restart: always
     environment:
       EDITION: SELF_HOSTED
@@ -206,6 +206,11 @@ services:
       - ./volumes/db/data:/var/lib/postgresql/data
     ports:
       - "5432:5432"
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 1s
+      timeout: 3s
+      retries: 30
 
   # The redis cache.
   redis:
@@ -216,6 +221,8 @@ services:
       - ./volumes/redis/data:/data
     # Set the redis password when startup redis server.
     command: redis-server --requirepass difyai123456
+    healthcheck:
+      test: ["CMD", "redis-cli","ping"]
 
   # The Weaviate vector store.
   weaviate:

web/README.md

@@ -2,41 +2,91 @@
 This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
 
 ## Getting Started
+### Run by source code
+To start the web frontend service, you will need [Node.js v18.x (LTS)](https://nodejs.org/en) and [NPM version 8.x.x](https://www.npmjs.com/) or [Yarn](https://yarnpkg.com/).
 
-First, run the development server:
+First, install the dependencies:
+
+```bash
+npm install
+# or
+yarn
+```
+
+Then, configure the environment variables. Create a file named `.env.local` in the current directory and copy the contents from `.env.example`. Modify the values of these environment variables according to your requirements:
+```
+# For production release, change this to PRODUCTION
+NEXT_PUBLIC_DEPLOY_ENV=DEVELOPMENT
+# The deployment edition, SELF_HOSTED or CLOUD
+NEXT_PUBLIC_EDITION=SELF_HOSTED
+# The base URL of console application, refers to the Console base URL of WEB service if console domain is
+# different from api or web app domain.
+# example: http://cloud.dify.ai/console/api
+NEXT_PUBLIC_API_PREFIX=http://localhost:5001/console/api
+# The URL for Web APP, refers to the Web App base URL of WEB service if web app domain is different from
+# console or api domain.
+# example: http://udify.app/api
+NEXT_PUBLIC_PUBLIC_API_PREFIX=http://localhost:5001/api
+
+# SENTRY
+NEXT_PUBLIC_SENTRY_DSN=
+```
+
+Finally, run the development server:
 
 ```bash
 npm run dev
 # or
 yarn dev
-# or
-pnpm dev
 ```
 
 Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
 
-You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+You can start editing the file under folder `app`. The page auto-updates as you edit the file.
 
-[API routes](https://nextjs.org/docs/api-routes/introduction) can be accessed on [http://localhost:3000/api/hello](http://localhost:3000/api/hello). This endpoint can be edited in `pages/api/hello.ts`.
+### Run by Docker
+First, Build the frontend image：
+```bash
+docker build . -t dify-web
+```
 
-The `pages/api` directory is mapped to `/api/*`. Files in this directory are treated as [API routes](https://nextjs.org/docs/api-routes/introduction) instead of React pages.
+Then, configure the environment variables.Use the same method mentioned in run by source code.
 
-This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+Finally, run the frontend service:
+```bash
+docker run -it -p 3000:3000 -e EDITION=SELF_HOSTED -e CONSOLE_URL=http://127.0.0.1:3000 -e APP_URL=http://127.0.0.1:3000 dify-web
+```
+
+When the console api domain and web app api domain are different, you can set the CONSOLE_URL and APP_URL separately.
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+## Deploy
+### Deploy on server
+First, build the app for production:
+
+```bash
+npm run build
+```
+
+Then, move the static files to standalone folder:
+```bash
+mv .next/static .next/standalone/.next
+cp -r ./public .next/standalone/.next/
+```
+
+Finally, start the app:
+```bash
+node .next/standalone/server.js 
+```
+
+If your project needs alternative port or hostname for listening, you can define PORT and HOSTNAME environment variables, before running server.js. For example, `PORT=3000 HOSTNAME=localhost node .next/standalone/server.js`.
 
 ## Lint Code
-If your ide is VSCode, rename `web/.vscode/settings.example.json` to `web/.vscode/settings.json` for lint code setting.
+If your IDE is VSCode, rename `web/.vscode/settings.example.json` to `web/.vscode/settings.json` for lint code setting.
 
-## Learn More
+## Documentation
+Visit https://docs.dify.ai/getting-started/readme to view the full documentation.
 
-To learn more about Next.js, take a look at the following resources:
-
-- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
-- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
-
-You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
-
-## Deploy on Vercel
-
-The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
-
-Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.
+## Community
+The Dify community can be found on [Discord community](https://discord.com/invite/FngNHpbcY7), where you can ask questions, voice ideas, and share your projects.

web/app/(commonLayout)/app/(appDetailLayout)/[appId]/overview/cardView.tsx

@@ -33,20 +33,18 @@ const CardView: FC<ICardViewProps> = ({ appId }) => {
   if (!response)
     return <Loading />
 
-  const handleError = (err: Error | null) => {
-    if (!err) {
-      notify({
-        type: 'success',
-        message: t('common.actionMsg.modifiedSuccessfully'),
-      })
+  const handleCallbackResult = (err: Error | null, message?: string) => {
+    const type = err ? 'error' : 'success'
+
+    message ||= (type === 'success' ? 'modifiedSuccessfully' : 'modifiedUnsuccessfully')
+
+    if (type === 'success') {
       mutate(detailParams)
     }
-    else {
-      notify({
-        type: 'error',
-        message: t('common.actionMsg.modificationFailed'),
-      })
-    }
+    notify({
+      type,
+      message: t(`common.actionMsg.${message}`),
+    })
   }
 
   const onChangeSiteStatus = async (value: boolean) => {
@@ -56,7 +54,8 @@ const CardView: FC<ICardViewProps> = ({ appId }) => {
         body: { enable_site: value },
       }) as Promise<App>,
     )
-    handleError(err)
+
+    handleCallbackResult(err)
   }
 
   const onChangeApiStatus = async (value: boolean) => {
@@ -66,7 +65,8 @@ const CardView: FC<ICardViewProps> = ({ appId }) => {
         body: { enable_api: value },
       }) as Promise<App>,
     )
-    handleError(err)
+
+    handleCallbackResult(err)
   }
 
   const onSaveSiteConfig: IAppCardProps['onSaveSiteConfig'] = async (params) => {
@@ -79,7 +79,7 @@ const CardView: FC<ICardViewProps> = ({ appId }) => {
     if (!err)
       localStorage.setItem(NEED_REFRESH_APP_LIST_KEY, '1')
 
-    handleError(err)
+      handleCallbackResult(err)
   }
 
   const onGenerateCode = async () => {
@@ -88,7 +88,8 @@ const CardView: FC<ICardViewProps> = ({ appId }) => {
         url: `/apps/${appId}/site/access-token-reset`,
       }) as Promise<UpdateAppSiteCodeResponse>,
     )
-    handleError(err)
+
+    handleCallbackResult(err, err ? 'generatedUnsuccessfully' : 'generatedSuccessfully')
   }
 
   return (

web/app/(commonLayout)/apps/AppCard.tsx

@@ -16,6 +16,7 @@ import { ToastContext } from '@/app/components/base/toast'
 import { deleteApp, fetchAppDetail, updateAppSiteConfig } from '@/service/apps'
 import AppIcon from '@/app/components/base/app-icon'
 import AppsContext, { useAppContext } from '@/context/app-context'
+import type { HtmlContentProps } from '@/app/components/base/popover'
 import CustomPopover from '@/app/components/base/popover'
 import Divider from '@/app/components/base/divider'
 import { asyncRunSafe } from '@/utils'
@@ -93,21 +94,21 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => {
       else {
         notify({
           type: 'error',
-          message: t('common.actionMsg.modificationFailed'),
+          message: t('common.actionMsg.modifiedUnsuccessfully'),
         })
       }
     },
     [app.id],
   )
 
-  const Operations = (props: any) => {
+  const Operations = (props: HtmlContentProps) => {
     const onClickSettings = async (e: React.MouseEvent<HTMLButtonElement>) => {
-      props?.onClose()
+      props.onClick?.()
       e.preventDefault()
       await getAppDetail()
     }
     const onClickDelete = async (e: React.MouseEvent<HTMLDivElement>) => {
-      props?.onClose()
+      props.onClick?.()
       e.preventDefault()
       setShowConfirmDelete(true)
     }
@@ -157,6 +158,7 @@ const AppCard = ({ app, onRefresh }: AppCardProps) => {
               )
             }
             className={'!w-[128px] h-fit !z-20'}
+            manualClose
           />}
         </div>
         <div className={style.listItemDescription}>

web/app/activate/activateForm.tsx

@@ -31,8 +31,8 @@ const ActivateForm = () => {
   const checkParams = {
     url: '/activate/check',
     params: {
-      workspace_id: workspaceID,
-      email,
+      ...workspaceID && { workspace_id: workspaceID },
+      ...email && { email },
       token,
     },
   }

web/app/components/app/chat/answer/index.tsx

@@ -5,7 +5,7 @@ import { useTranslation } from 'react-i18next'
 import { useContext } from 'use-context-selector'
 import { UserCircleIcon } from '@heroicons/react/24/solid'
 import cn from 'classnames'
-import type { DisplayScene, FeedbackFunc, Feedbacktype, IChatItem, SubmitAnnotationFunc, ThoughtItem } from '../type'
+import type { CitationItem, DisplayScene, FeedbackFunc, Feedbacktype, IChatItem, SubmitAnnotationFunc, ThoughtItem } from '../type'
 import OperationBtn from '../operation'
 import LoadingAnim from '../loading-anim'
 import { EditIcon, EditIconSolid, OpeningStatementIcon, RatingIcon } from '../icon-component'
@@ -13,6 +13,7 @@ import s from '../style.module.css'
 import MoreInfo from '../more-info'
 import CopyBtn from '../copy-btn'
 import Thought from '../thought'
+import Citation from '../citation'
 import { randomString } from '@/utils'
 import type { Annotation, MessageRating } from '@/models/log'
 import AppContext from '@/context/app-context'
@@ -45,11 +46,14 @@ export type IAnswerProps = {
   isResponsing?: boolean
   answerIconClassName?: string
   thoughts?: ThoughtItem[]
+  citation?: CitationItem[]
   isThinking?: boolean
   dataSets?: DataSet[]
+  isShowCitation?: boolean
+  isShowCitationHitInfo?: boolean
 }
 // The component needs to maintain its own state to control whether to display input component
-const Answer: FC<IAnswerProps> = ({ item, feedbackDisabled = false, isHideFeedbackEdit = false, onFeedback, onSubmitAnnotation, displayScene = 'web', isResponsing, answerIconClassName, thoughts, isThinking, dataSets }) => {
+const Answer: FC<IAnswerProps> = ({ item, feedbackDisabled = false, isHideFeedbackEdit = false, onFeedback, onSubmitAnnotation, displayScene = 'web', isResponsing, answerIconClassName, thoughts, citation, isThinking, dataSets, isShowCitation, isShowCitationHitInfo = false }) => {
   const { id, content, more, feedback, adminFeedback, annotation: initAnnotation } = item
   const [showEdit, setShowEdit] = useState(false)
   const [loading, setLoading] = useState(false)
@@ -171,7 +175,7 @@ const Answer: FC<IAnswerProps> = ({ item, feedbackDisabled = false, isHideFeedba
             </div>
           }
         </div>
-        <div className={s.answerWrapWrap}>
+        <div className={cn(s.answerWrapWrap, 'chat-answer-container')}>
           <div className={`${s.answerWrap} ${showEdit ? 'w-full' : ''}`}>
             <div className={`${s.answer} relative text-sm text-gray-900`}>
               <div className={'ml-2 py-3 px-4 bg-gray-100 rounded-tr-2xl rounded-b-2xl'}>
@@ -237,6 +241,11 @@ const Answer: FC<IAnswerProps> = ({ item, feedbackDisabled = false, isHideFeedba
                     </div>
                   </>
                 }
+                {
+                  !!citation?.length && !isThinking && isShowCitation && !isResponsing && (
+                    <Citation data={citation} showHitInfo={isShowCitationHitInfo} />
+                  )
+                }
               </div>
               <div className='absolute top-[-14px] right-[-14px] flex flex-row justify-end gap-1'>
                 {!item.isOpeningStatement && (

web/app/components/app/chat/citation/index.tsx

@@ -0,0 +1,123 @@
+import { useEffect, useMemo, useRef, useState } from 'react'
+import type { FC } from 'react'
+import { useTranslation } from 'react-i18next'
+import type { CitationItem } from '../type'
+import Popup from './popup'
+import { ChevronDown } from '@/app/components/base/icons/src/vender/line/arrows'
+
+export type Resources = {
+  documentId: string
+  documentName: string
+  dataSourceType: string
+  sources: CitationItem[]
+}
+
+type CitationProps = {
+  data: CitationItem[]
+  showHitInfo?: boolean
+}
+const Citation: FC<CitationProps> = ({
+  data,
+  showHitInfo,
+}) => {
+  const { t } = useTranslation()
+  const elesRef = useRef<HTMLDivElement[]>([])
+  const [limitNumberInOneLine, setlimitNumberInOneLine] = useState(0)
+  const [showMore, setShowMore] = useState(false)
+  const resources = useMemo(() => data.reduce((prev: Resources[], next) => {
+    const documentId = next.document_id
+    const documentName = next.document_name
+    const dataSourceType = next.data_source_type
+    const documentIndex = prev.findIndex(i => i.documentId === documentId)
+
+    if (documentIndex > -1) {
+      prev[documentIndex].sources.push(next)
+    }
+    else {
+      prev.push({
+        documentId,
+        documentName,
+        dataSourceType,
+        sources: [next],
+      })
+    }
+
+    return prev
+  }, []), [data])
+
+  const handleAdjustResourcesLayout = () => {
+    const containerWidth = document.querySelector('.chat-answer-container')!.clientWidth - 40
+    let totalWidth = 0
+    for (let i = 0; i < resources.length; i++) {
+      totalWidth += elesRef.current[i].clientWidth
+
+      if (totalWidth + i * 4 > containerWidth!) {
+        totalWidth -= elesRef.current[i].clientWidth
+
+        if (totalWidth + 34 > containerWidth!)
+          setlimitNumberInOneLine(i - 1)
+        else
+          setlimitNumberInOneLine(i)
+
+        break
+      }
+      else {
+        setlimitNumberInOneLine(i + 1)
+      }
+    }
+  }
+
+  useEffect(() => {
+    handleAdjustResourcesLayout()
+  }, [])
+
+  const resourcesLength = resources.length
+
+  return (
+    <div className='mt-3 -mb-1'>
+      <div className='flex items-center mb-2 text-xs font-medium text-gray-500'>
+        {t('common.chat.citation.title')}
+        <div className='grow ml-2 h-[1px] bg-black/5' />
+      </div>
+      <div className='relative flex flex-wrap'>
+        {
+          resources.map((res, index) => (
+            <div
+              key={index}
+              className='absolute top-0 left-0 w-auto mr-1 mb-1 pl-7 pr-2 max-w-[240px] h-7 text-xs whitespace-nowrap opacity-0 -z-10'
+              ref={ele => (elesRef.current[index] = ele!)}
+            >
+              {res.documentName}
+            </div>
+          ))
+        }
+        {
+          resources.slice(0, showMore ? resourcesLength : limitNumberInOneLine).map((res, index) => (
+            <div key={index} className='mr-1 mb-1 cursor-pointer'>
+              <Popup
+                data={res}
+                showHitInfo={showHitInfo}
+              />
+            </div>
+          ))
+        }
+        {
+          limitNumberInOneLine < resourcesLength && (
+            <div
+              className='flex items-center px-2 h-7 bg-white rounded-lg text-xs font-medium text-gray-500 cursor-pointer'
+              onClick={() => setShowMore(v => !v)}
+            >
+              {
+                !showMore
+                  ? `+ ${resourcesLength - limitNumberInOneLine}`
+                  : <ChevronDown className='w-4 h-4 text-gray-600 rotate-180' />
+              }
+            </div>
+          )
+        }
+      </div>
+    </div>
+  )
+}
+
+export default Citation

web/app/components/app/chat/citation/popup.tsx

@@ -0,0 +1,123 @@
+import { Fragment, useState } from 'react'
+import type { FC } from 'react'
+import Link from 'next/link'
+import { useTranslation } from 'react-i18next'
+import Tooltip from './tooltip'
+import ProgressTooltip from './progress-tooltip'
+import type { Resources } from './index'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+  PortalToFollowElemTrigger,
+} from '@/app/components/base/portal-to-follow-elem'
+import FileIcon from '@/app/components/base/file-icon'
+import {
+  Hash02,
+  Target04,
+} from '@/app/components/base/icons/src/vender/line/general'
+import { ArrowUpRight } from '@/app/components/base/icons/src/vender/line/arrows'
+import {
+  BezierCurve03,
+  TypeSquare,
+} from '@/app/components/base/icons/src/vender/line/editor'
+
+type PopupProps = {
+  data: Resources
+  showHitInfo?: boolean
+}
+
+const Popup: FC<PopupProps> = ({
+  data,
+  showHitInfo = false,
+}) => {
+  const { t } = useTranslation()
+  const [open, setOpen] = useState(false)
+  const fileType = data.dataSourceType === 'upload_file'
+    ? (/\.([^.]*)$/g.exec(data.documentName)?.[1] || '')
+    : 'notion'
+
+  return (
+    <PortalToFollowElem
+      open={open}
+      onOpenChange={setOpen}
+      placement='top-start'
+      offset={{
+        mainAxis: 8,
+        crossAxis: -2,
+      }}
+    >
+      <PortalToFollowElemTrigger onClick={() => setOpen(v => !v)}>
+        <div className='flex items-center px-2 max-w-[240px] h-7 bg-white rounded-lg'>
+          <FileIcon type={fileType} className='mr-1 w-4 h-4' />
+          <div className='text-xs text-gray-600 truncate'>{data.documentName}</div>
+        </div>
+      </PortalToFollowElemTrigger>
+      <PortalToFollowElemContent style={{ zIndex: 1000 }}>
+        <div className='w-[360px] bg-gray-50 rounded-xl shadow-lg'>
+          <div className='px-4 pt-3 pb-2'>
+            <div className='flex items-center h-[18px]'>
+              <FileIcon type={fileType} className='mr-1 w-4 h-4' />
+              <div className='text-xs font-medium text-gray-600 truncate'>{data.documentName}</div>
+            </div>
+          </div>
+          <div className='px-4 py-0.5 max-h-[450px] bg-white rounded-lg overflow-auto'>
+            {
+              data.sources.map((source, index) => (
+                <Fragment key={index}>
+                  <div className='group py-3'>
+                    {
+                      showHitInfo && (
+                        <div className='flex items-center justify-between mb-2'>
+                          <div className='flex items-center px-1.5 h-5 border border-gray-200 rounded-md'>
+                            <Hash02 className='mr-0.5 w-3 h-3 text-gray-400' />
+                            <div className='text-[11px] font-medium text-gray-500'>{source.segment_position}</div>
+                          </div>
+                          <Link
+                            href={`/datasets/${source.dataset_id}/documents/${source.document_id}`}
+                            className='hidden items-center h-[18px] text-xs text-primary-600 group-hover:flex'>
+                            Link to dataset
+                            <ArrowUpRight className='ml-1 w-3 h-3' />
+                          </Link>
+                        </div>
+                      )
+                    }
+                    <div className='text-[13px] text-gray-800'>{source.content}</div>
+                    {
+                      showHitInfo && (
+                        <div className='flex items-center mt-2 text-xs font-medium text-gray-500'>
+                          <Tooltip
+                            text={t('common.chat.citation.characters')}
+                            data={source.word_count}
+                            icon={<TypeSquare className='mr-1 w-3 h-3' />}
+                          />
+                          <Tooltip
+                            text={t('common.chat.citation.hitCount')}
+                            data={source.hit_count}
+                            icon={<Target04 className='mr-1 w-3 h-3' />}
+                          />
+                          <Tooltip
+                            text={t('common.chat.citation.vectorHash')}
+                            data={source.index_node_hash.substring(0, 7)}
+                            icon={<BezierCurve03 className='mr-1 w-3 h-3' />}
+                          />
+                          <ProgressTooltip data={Number(source.score.toFixed(2))} />
+                        </div>
+                      )
+                    }
+                  </div>
+                  {
+                    index !== data.sources.length - 1 && (
+                      <div className='my-1 h-[1px] bg-black/5' />
+                    )
+                  }
+                </Fragment>
+              ))
+            }
+          </div>
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+
+export default Popup

web/app/components/app/chat/citation/progress-tooltip.tsx

@@ -0,0 +1,46 @@
+import { useState } from 'react'
+import type { FC } from 'react'
+import { useTranslation } from 'react-i18next'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+  PortalToFollowElemTrigger,
+} from '@/app/components/base/portal-to-follow-elem'
+
+type ProgressTooltipProps = {
+  data: number
+}
+
+const ProgressTooltip: FC<ProgressTooltipProps> = ({
+  data,
+}) => {
+  const { t } = useTranslation()
+  const [open, setOpen] = useState(false)
+
+  return (
+    <PortalToFollowElem
+      open={open}
+      onOpenChange={setOpen}
+      placement='top-start'
+    >
+      <PortalToFollowElemTrigger
+        onMouseEnter={() => setOpen(true)}
+        onMouseLeave={() => setOpen(false)}
+      >
+        <div className='grow flex items-center'>
+          <div className='mr-1 w-16 h-1.5 rounded-[3px] border border-gray-400 overflow-hidden'>
+            <div className='bg-gray-400 h-full' style={{ width: `${data * 100}%` }}></div>
+          </div>
+          {data}
+        </div>
+      </PortalToFollowElemTrigger>
+      <PortalToFollowElemContent style={{ zIndex: 1001 }}>
+        <div className='p-3 bg-white text-xs font-medium text-gray-500 rounded-lg shadow-lg'>
+          {t('common.chat.citation.hitScore')} {data}
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+
+export default ProgressTooltip

web/app/components/app/chat/citation/tooltip.tsx

@@ -0,0 +1,46 @@
+import React, { useState } from 'react'
+import type { FC } from 'react'
+import {
+  PortalToFollowElem,
+  PortalToFollowElemContent,
+  PortalToFollowElemTrigger,
+} from '@/app/components/base/portal-to-follow-elem'
+
+type TooltipProps = {
+  data: number | string
+  text: string
+  icon: React.ReactNode
+}
+
+const Tooltip: FC<TooltipProps> = ({
+  data,
+  text,
+  icon,
+}) => {
+  const [open, setOpen] = useState(false)
+
+  return (
+    <PortalToFollowElem
+      open={open}
+      onOpenChange={setOpen}
+      placement='top-start'
+    >
+      <PortalToFollowElemTrigger
+        onMouseEnter={() => setOpen(true)}
+        onMouseLeave={() => setOpen(false)}
+      >
+        <div className='flex items-center mr-6'>
+          {icon}
+          {data}
+        </div>
+      </PortalToFollowElemTrigger>
+      <PortalToFollowElemContent style={{ zIndex: 1001 }}>
+        <div className='p-3 bg-white text-xs font-medium text-gray-500 rounded-lg shadow-lg'>
+          {text} {data}
+        </div>
+      </PortalToFollowElemContent>
+    </PortalToFollowElem>
+  )
+}
+
+export default Tooltip

web/app/components/app/chat/index.tsx

@@ -48,15 +48,17 @@ export type IChatProps = {
   isShowSuggestion?: boolean
   suggestionList?: string[]
   isShowSpeechToText?: boolean
+  isShowCitation?: boolean
   answerIconClassName?: string
   isShowConfigElem?: boolean
   dataSets?: DataSet[]
+  isShowCitationHitInfo?: boolean
 }
 
 const Chat: FC<IChatProps> = ({
   configElem,
   chatList,
-  controlChatUpdateAllConversation,
+
   feedbackDisabled = false,
   isHideFeedbackEdit = false,
   isHideSendInput = false,
@@ -74,9 +76,11 @@ const Chat: FC<IChatProps> = ({
   isShowSuggestion,
   suggestionList,
   isShowSpeechToText,
+  isShowCitation,
   answerIconClassName,
   isShowConfigElem,
   dataSets,
+  isShowCitationHitInfo,
 }) => {
   const { t } = useTranslation()
   const { notify } = useContext(ToastContext)
@@ -162,6 +166,7 @@ const Chat: FC<IChatProps> = ({
           if (item.isAnswer) {
             const isLast = item.id === chatList[chatList.length - 1].id
             const thoughts = item.agent_thoughts?.filter(item => item.thought !== '[DONE]')
+            const citation = item.citation
             const isThinking = !item.content && item.agent_thoughts && item.agent_thoughts?.length > 0 && !item.agent_thoughts.some(item => item.thought === '[DONE]')
             return <Answer
               key={item.id}
@@ -174,8 +179,11 @@ const Chat: FC<IChatProps> = ({
               isResponsing={isResponsing && isLast}
               answerIconClassName={answerIconClassName}
               thoughts={thoughts}
+              citation={citation}
               isThinking={isThinking}
               dataSets={dataSets}
+              isShowCitation={isShowCitation}
+              isShowCitationHitInfo={isShowCitationHitInfo}
             />
           }
           return <Question key={item.id} id={item.id} content={item.content} more={item.more} useCurrentUserAvatar={useCurrentUserAvatar} />

web/app/components/app/chat/type.ts

@@ -23,10 +23,26 @@ export type ThoughtItem = {
   tool_input: string
   message_id: string
 }
+export type CitationItem = {
+  content: string
+  data_source_type: string
+  dataset_name: string
+  dataset_id: string
+  document_id: string
+  document_name: string
+  hit_count: number
+  index_node_hash: string
+  segment_id: string
+  segment_position: number
+  score: number
+  word_count: number
+}
+
 export type IChatItem = {
   id: string
   content: string
   agent_thoughts?: ThoughtItem[]
+  citation?: CitationItem[]
   /**
    * Specific message type
    */
@@ -51,3 +67,8 @@ export type IChatItem = {
   useCurrentUserAvatar?: boolean
   isOpeningStatement?: boolean
 }
+
+export type MessageEnd = {
+  id: string
+  retriever_resources?: CitationItem[]
+}

web/app/components/app/configuration/base/icons/remove-icon/index.tsx

@@ -1,18 +1,18 @@
 'use client'
-import React, { FC, useState } from 'react'
+import React, { useState } from 'react'
 import cn from 'classnames'
 
-export interface IRemoveIconProps {
+type IRemoveIconProps = {
   className?: string
   isHoverStatus?: boolean
   onClick: () => void
 }
 
-const RemoveIcon: FC<IRemoveIconProps> = ({
+const RemoveIcon = ({
   className,
   isHoverStatus,
-  onClick
-}) => {
+  onClick,
+}: IRemoveIconProps) => {
   const [isHovered, setIsHovered] = useState(false)
   const computedIsHovered = isHoverStatus || isHovered
   return (
@@ -23,7 +23,7 @@ const RemoveIcon: FC<IRemoveIconProps> = ({
       onClick={onClick}
     >
       <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
-        <path d="M10 6H14M6 8H18M16.6667 8L16.1991 15.0129C16.129 16.065 16.0939 16.5911 15.8667 16.99C15.6666 17.3412 15.3648 17.6235 15.0011 17.7998C14.588 18 14.0607 18 13.0062 18H10.9938C9.93927 18 9.41202 18 8.99889 17.7998C8.63517 17.6235 8.33339 17.3412 8.13332 16.99C7.90607 16.5911 7.871 16.065 7.80086 15.0129L7.33333 8M10.6667 11V14.3333M13.3333 11V14.3333" stroke={computedIsHovered ? "#D92D20" : "#667085"} strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
+        <path d="M10 6H14M6 8H18M16.6667 8L16.1991 15.0129C16.129 16.065 16.0939 16.5911 15.8667 16.99C15.6666 17.3412 15.3648 17.6235 15.0011 17.7998C14.588 18 14.0607 18 13.0062 18H10.9938C9.93927 18 9.41202 18 8.99889 17.7998C8.63517 17.6235 8.33339 17.3412 8.13332 16.99C7.90607 16.5911 7.871 16.065 7.80086 15.0129L7.33333 8M10.6667 11V14.3333M13.3333 11V14.3333" stroke={computedIsHovered ? '#D92D20' : '#667085'} strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
       </svg>
     </div>
   )

web/app/components/app/configuration/config-var/config-modal/index.tsx

@@ -2,6 +2,7 @@
 import type { FC } from 'react'
 import React, { useEffect, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import { useContext } from 'use-context-selector'
 import ModalFoot from '../modal-foot'
 import type { Options } from '../config-select'
 import ConfigSelect from '../config-select'
@@ -11,6 +12,7 @@ import s from './style.module.css'
 import Toast from '@/app/components/base/toast'
 import type { PromptVariable } from '@/models/debug'
 import { getNewVar } from '@/utils/var'
+import ConfigContext from '@/context/debug-configuration'
 
 import Modal from '@/app/components/base/modal'
 
@@ -28,6 +30,7 @@ const ConfigModal: FC<IConfigModalProps> = ({
   onClose,
   onConfirm,
 }) => {
+  const { modelConfig } = useContext(ConfigContext)
   const { t } = useTranslation()
   const { type, name, key, options, max_length } = payload || getNewVar('')
 
@@ -41,7 +44,7 @@ const ConfigModal: FC<IConfigModalProps> = ({
     }
   }
 
-  const isStringInput = tempType === 'string'
+  const isStringInput = tempType === 'string' || tempType === 'paragraph'
   const title = isStringInput ? t('appDebug.variableConig.maxLength') : t('appDebug.variableConig.options')
 
   // string type
@@ -93,22 +96,24 @@ const ConfigModal: FC<IConfigModalProps> = ({
         <div className='mb-2'>
           <div className={s.title}>{t('appDebug.variableConig.fieldType')}</div>
           <div className='flex space-x-2'>
-            <SelectTypeItem type='string' selected={isStringInput} onClick={handleTypeChange('string')} />
-            <SelectTypeItem type='select' selected={!isStringInput} onClick={handleTypeChange('select')} />
+            <SelectTypeItem type='string' selected={tempType === 'string'} onClick={handleTypeChange('string')} />
+            <SelectTypeItem type='paragraph' selected={tempType === 'paragraph'} onClick={handleTypeChange('paragraph')} />
+            <SelectTypeItem type='select' selected={tempType === 'select'} onClick={handleTypeChange('select')} />
           </div>
         </div>
 
-        <div className='mt-6'>
-          <div className={s.title}>{title}</div>
-          {isStringInput
-            ? (
-              <ConfigString value={tempMaxLength} onChange={setTempMaxValue} />
-            )
-            : (
-              <ConfigSelect options={tempOptions} onChange={setTempOptions} />
-            )}
-        </div>
-
+        {tempType !== 'paragraph' && (
+          <div className='mt-6'>
+            <div className={s.title}>{title}</div>
+            {isStringInput
+              ? (
+                <ConfigString modelId={modelConfig.model_id} value={tempMaxLength} onChange={setTempMaxValue} />
+              )
+              : (
+                <ConfigSelect options={tempOptions} onChange={setTempOptions} />
+              )}
+          </div>
+        )}
       </div>
       <ModalFoot
         onConfirm={handleConfirm}

web/app/components/app/configuration/config-var/config-string/index.tsx

@@ -1,9 +1,10 @@
 'use client'
 import type { FC } from 'react'
-import React from 'react'
+import React, { useEffect } from 'react'
 
 export type IConfigStringProps = {
   value: number | undefined
+  modelId: string
   onChange: (value: number | undefined) => void
 }
 
@@ -13,6 +14,11 @@ const ConfigString: FC<IConfigStringProps> = ({
   value,
   onChange,
 }) => {
+  useEffect(() => {
+    if (value && value > MAX_LENGTH)
+      onChange(MAX_LENGTH)
+  }, [value, MAX_LENGTH])
+
   return (
     <div>
       <input
@@ -21,7 +27,13 @@ const ConfigString: FC<IConfigStringProps> = ({
         min={1}
         value={value || ''}
         onChange={(e) => {
-          const value = Math.max(1, Math.min(MAX_LENGTH, parseInt(e.target.value))) || 1
+          let value = parseInt(e.target.value, 10)
+          if (value > MAX_LENGTH)
+            value = MAX_LENGTH
+
+          else if (value < 1)
+            value = 1
+
           onChange(value)
         }}
         className="w-full px-3 text-sm leading-9 text-gray-900 border-0 rounded-lg grow h-9 bg-gray-50 focus:outline-none focus:ring-1 focus:ring-inset focus:ring-gray-200"

web/app/components/app/configuration/config-var/index.tsx

@@ -8,7 +8,7 @@ import type { Timeout } from 'ahooks/lib/useRequest/src/types'
 import Panel from '../base/feature-panel'
 import OperationBtn from '../base/operation-btn'
 import VarIcon from '../base/icons/var-icon'
-import EditModel from './config-model'
+import EditModal from './config-modal'
 import IconTypeIcon from './input-type-icon'
 import type { IInputTypeIconProps } from './input-type-icon'
 import s from './style.module.css'
@@ -52,13 +52,18 @@ const ConfigVar: FC<IConfigVarProps> = ({ promptVariables, readonly, onPromptVar
     onPromptVariablesChange?.(newPromptVariables)
   }
 
-  const batchUpdatePromptVariable = (key: string, updateKeys: string[], newValues: any[]) => {
+  const batchUpdatePromptVariable = (key: string, updateKeys: string[], newValues: any[], isParagraph?: boolean) => {
     const newPromptVariables = promptVariables.map((item) => {
       if (item.key === key) {
         const newItem: any = { ...item }
         updateKeys.forEach((updateKey, i) => {
           newItem[updateKey] = newValues[i]
         })
+        if (isParagraph) {
+          delete newItem.max_length
+          delete newItem.options
+        }
+        console.log(newItem)
         return newItem
       }
 
@@ -240,16 +245,15 @@ const ConfigVar: FC<IConfigVarProps> = ({ promptVariables, readonly, onPromptVar
       )}
 
       {isShowEditModal && (
-        <EditModel
+        <EditModal
           payload={currItem as PromptVariable}
           isShow={isShowEditModal}
           onClose={hideEditModal}
           onConfirm={({ type, value }) => {
             if (type === 'string')
               batchUpdatePromptVariable(currKey as string, ['type', 'max_length'], [type, value || DEFAULT_VALUE_MAX_LEN])
-
             else
-              batchUpdatePromptVariable(currKey as string, ['type', 'options'], [type, value || []])
+              batchUpdatePromptVariable(currKey as string, ['type', 'options'], [type, value || []], type === 'paragraph')
 
             hideEditModal()
           }}

web/app/components/app/configuration/config-var/input-type-icon.tsx

@@ -13,6 +13,14 @@ const IconMap = (type: IInputTypeIconProps['type']) => {
         <path fillRule="evenodd" clipRule="evenodd" d="M3.52593 0.166672H8.47411C8.94367 0.166665 9.33123 0.166659 9.64692 0.192452C9.97481 0.219242 10.2762 0.276738 10.5593 0.420991C10.9984 0.644695 11.3553 1.00165 11.579 1.44069C11.7233 1.72381 11.7808 2.02522 11.8076 2.35311C11.8334 2.6688 11.8334 3.05634 11.8334 3.5259V8.47411C11.8334 8.94367 11.8334 9.33121 11.8076 9.6469C11.7808 9.97479 11.7233 10.2762 11.579 10.5593C11.3553 10.9984 10.9984 11.3553 10.5593 11.579C10.2762 11.7233 9.97481 11.7808 9.64692 11.8076C9.33123 11.8334 8.94369 11.8333 8.47413 11.8333H3.52592C3.05636 11.8333 2.66882 11.8334 2.35312 11.8076C2.02523 11.7808 1.72382 11.7233 1.44071 11.579C1.00167 11.3553 0.644711 10.9984 0.421006 10.5593C0.276753 10.2762 0.219257 9.97479 0.192468 9.6469C0.166674 9.33121 0.16668 8.94366 0.166687 8.4741V3.52591C0.16668 3.05635 0.166674 2.6688 0.192468 2.35311C0.219257 2.02522 0.276753 1.72381 0.421006 1.44069C0.644711 1.00165 1.00167 0.644695 1.44071 0.420991C1.72382 0.276738 2.02523 0.219242 2.35312 0.192452C2.66882 0.166659 3.05637 0.166665 3.52593 0.166672ZM3.08335 3.08334C3.08335 2.76117 3.34452 2.50001 3.66669 2.50001H8.33335C8.65552 2.50001 8.91669 2.76117 8.91669 3.08334C8.91669 3.4055 8.65552 3.66667 8.33335 3.66667H6.58335V8.91667C6.58335 9.23884 6.32219 9.5 6.00002 9.5C5.67785 9.5 5.41669 9.23884 5.41669 8.91667V3.66667H3.66669C3.34452 3.66667 3.08335 3.4055 3.08335 3.08334Z" fill="#98A2B3" />
       </svg>
     ),
+    paragraph: (
+      <svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg">
+        <path fillRule="evenodd" clipRule="evenodd" d="M1.16669 5.83329C1.16669 5.51113 1.42785 5.24996 1.75002 5.24996H9.33335C9.65552 5.24996 9.91669 5.51113 9.91669 5.83329C9.91669 6.15546 9.65552 6.41663 9.33335 6.41663H1.75002C1.42785 6.41663 1.16669 6.15546 1.16669 5.83329Z" fill="#98A2B3"/>
+        <path fillRule="evenodd" clipRule="evenodd" d="M1.16669 3.49996C1.16669 3.17779 1.42785 2.91663 1.75002 2.91663H11.6667C11.9889 2.91663 12.25 3.17779 12.25 3.49996C12.25 3.82213 11.9889 4.08329 11.6667 4.08329H1.75002C1.42785 4.08329 1.16669 3.82213 1.16669 3.49996Z" fill="#98A2B3"/>
+        <path fillRule="evenodd" clipRule="evenodd" d="M1.16669 8.16663C1.16669 7.84446 1.42785 7.58329 1.75002 7.58329H11.6667C11.9889 7.58329 12.25 7.84446 12.25 8.16663C12.25 8.48879 11.9889 8.74996 11.6667 8.74996H1.75002C1.42785 8.74996 1.16669 8.48879 1.16669 8.16663Z" fill="#98A2B3"/>
+        <path fillRule="evenodd" clipRule="evenodd" d="M1.16669 10.5C1.16669 10.1778 1.42785 9.91663 1.75002 9.91663H9.33335C9.65552 9.91663 9.91669 10.1778 9.91669 10.5C9.91669 10.8221 9.65552 11.0833 9.33335 11.0833H1.75002C1.42785 11.0833 1.16669 10.8221 1.16669 10.5Z" fill="#98A2B3"/>
+      </svg>
+    ),
     select: (
       <svg width="14" height="14" viewBox="0 0 14 14" fill="none" xmlns="http://www.w3.org/2000/svg">
         <path fillRule="evenodd" clipRule="evenodd" d="M7.48913 4.08334H3.01083C2.70334 4.08333 2.43804 4.08333 2.21955 4.10118C1.98893 4.12002 1.75955 4.16162 1.53883 4.27408C1.20955 4.44186 0.941831 4.70958 0.774053 5.03886C0.66159 5.25958 0.619989 5.48896 0.601147 5.71958C0.583295 5.93807 0.583304 6.20334 0.583313 6.51084V10.9892C0.583304 11.2967 0.583295 11.5619 0.601147 11.7804C0.619989 12.0111 0.66159 12.2404 0.774053 12.4612C0.941831 12.7904 1.20955 13.0582 1.53883 13.2259C1.75955 13.3384 1.98893 13.38 2.21955 13.3988C2.43803 13.4167 2.70329 13.4167 3.01077 13.4167H7.48912C7.7966 13.4167 8.06193 13.4167 8.28041 13.3988C8.51103 13.38 8.74041 13.3384 8.96113 13.2259C9.29041 13.0582 9.55813 12.7904 9.72591 12.4612C9.83837 12.2404 9.87997 12.0111 9.89882 11.7804C9.91667 11.5619 9.91666 11.2967 9.91665 10.9892V6.51087C9.91666 6.20336 9.91667 5.93808 9.89882 5.71958C9.87997 5.48896 9.83837 5.25958 9.72591 5.03886C9.55813 4.70958 9.29041 4.44186 8.96113 4.27408C8.74041 4.16162 8.51103 4.12002 8.28041 4.10118C8.06192 4.08333 7.79663 4.08333 7.48913 4.08334ZM7.70413 7.70416C7.93193 7.47635 7.93193 7.107 7.70413 6.8792C7.47632 6.65139 7.10697 6.65139 6.87917 6.8792L4.66665 9.09172L3.91246 8.33753C3.68465 8.10973 3.31531 8.10973 3.0875 8.33753C2.8597 8.56534 2.8597 8.93468 3.0875 9.16249L4.25417 10.3292C4.48197 10.557 4.85132 10.557 5.07913 10.3292L7.70413 7.70416Z" fill="#98A2B3" />

web/app/components/app/configuration/config-var/select-type-item/index.tsx

@@ -1,11 +1,12 @@
 'use client'
-import React, { FC } from 'react'
+import type { FC } from 'react'
+import React from 'react'
 import { useTranslation } from 'react-i18next'
 import cn from 'classnames'
 
 import s from './style.module.css'
 
-export interface ISelectTypeItemProps {
+export type ISelectTypeItemProps = {
   type: string
   selected: boolean
   onClick: () => void
@@ -14,46 +15,75 @@ export interface ISelectTypeItemProps {
 const Icon = ({ type, selected }: Partial<ISelectTypeItemProps>) => {
   switch (type) {
     case 'select':
-      return selected ? (
-        <svg width="17" height="16" viewBox="0 0 17 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-          <path fillRule="evenodd" clipRule="evenodd" d="M8.89233 4.66669H3.77428C3.42285 4.66668 3.11966 4.66667 2.86995 4.68707C2.60639 4.7086 2.34424 4.75615 2.09199 4.88468C1.71567 5.07642 1.4097 5.38238 1.21796 5.75871C1.08943 6.01096 1.04188 6.27311 1.02035 6.53667C0.999949 6.78638 0.999959 7.08954 0.99997 7.44096V12.5591C0.999959 12.9105 0.999949 13.2137 1.02035 13.4634C1.04188 13.7269 1.08943 13.9891 1.21796 14.2413C1.4097 14.6177 1.71567 14.9236 2.09199 15.1154C2.34424 15.2439 2.60639 15.2914 2.86995 15.313C3.11965 15.3334 3.4228 15.3334 3.77421 15.3334H8.89232C9.24372 15.3334 9.54696 15.3334 9.79666 15.313C10.0602 15.2914 10.3224 15.2439 10.5746 15.1154C10.9509 14.9236 11.2569 14.6177 11.4487 14.2413C11.5772 13.9891 11.6247 13.7269 11.6463 13.4634C11.6667 13.2137 11.6667 12.9105 11.6666 12.559V7.44101C11.6667 7.08957 11.6667 6.78639 11.6463 6.53667C11.6247 6.27311 11.5772 6.01096 11.4487 5.75871C11.2569 5.38238 10.9509 5.07642 10.5746 4.88468C10.3224 4.75615 10.0602 4.7086 9.79666 4.68707C9.54695 4.66667 9.24376 4.66668 8.89233 4.66669ZM9.13804 8.80476C9.39839 8.54441 9.39839 8.1223 9.13804 7.86195C8.87769 7.6016 8.45558 7.6016 8.19523 7.86195L5.66664 10.3905L4.80471 9.52862C4.54436 9.26827 4.12225 9.26827 3.8619 9.52862C3.60155 9.78897 3.60155 10.2111 3.8619 10.4714L5.19523 11.8048C5.45558 12.0651 5.87769 12.0651 6.13804 11.8048L9.13804 8.80476Z" fill="#155EEF" />
-          <path d="M12.8923 0.666688H7.77427C7.42285 0.666676 7.11966 0.666666 6.86995 0.687068C6.60639 0.708602 6.34424 0.756146 6.09199 0.884676C5.71567 1.07642 5.40971 1.38238 5.21796 1.75871C5.08943 2.01096 5.04188 2.27311 5.02035 2.53667C5.00206 2.76053 5.00018 3.02734 4.99999 3.33337L8.92055 3.33336C9.2463 3.33329 9.59951 3.3332 9.90523 3.35818C10.2512 3.38645 10.7084 3.45642 11.1799 3.69668C11.8071 4.01626 12.3171 4.5262 12.6367 5.1534C12.8769 5.62495 12.9469 6.08209 12.9752 6.42811C13.0001 6.73384 13.0001 7.08704 13 7.4128L13 11.3333C13.306 11.3332 13.5728 11.3313 13.7967 11.313C14.0602 11.2914 14.3224 11.2439 14.5746 11.1154C14.9509 10.9236 15.2569 10.6177 15.4487 10.2413C15.5772 9.98908 15.6247 9.72694 15.6463 9.46338C15.6667 9.21368 15.6666 8.91052 15.6666 8.55912V3.44101C15.6666 3.0896 15.6667 2.78637 15.6463 2.53667C15.6247 2.27311 15.5772 2.01096 15.4487 1.75871C15.2569 1.38238 14.9509 1.07642 14.5746 0.884676C14.3224 0.756146 14.0602 0.708602 13.7967 0.687068C13.5469 0.666666 13.2438 0.666676 12.8923 0.666688Z" fill="#155EEF" />
-        </svg>
-
-      ) : (
-        <svg width="17" height="16" viewBox="0 0 17 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-          <path fillRule="evenodd" clipRule="evenodd" d="M8.89233 4.66667H3.77428C3.42285 4.66666 3.11966 4.66665 2.86995 4.68705C2.60639 4.70859 2.34424 4.75613 2.09199 4.88466C1.71567 5.07641 1.4097 5.38237 1.21796 5.75869C1.08943 6.01095 1.04188 6.27309 1.02035 6.53665C0.999949 6.78636 0.999959 7.08953 0.99997 7.44095V12.559C0.999959 12.9105 0.999949 13.2137 1.02035 13.4634C1.04188 13.7269 1.08943 13.9891 1.21796 14.2413C1.4097 14.6176 1.71567 14.9236 2.09199 15.1154C2.34424 15.2439 2.60639 15.2914 2.86995 15.313C3.11965 15.3334 3.4228 15.3334 3.77421 15.3333H8.89232C9.24372 15.3334 9.54696 15.3334 9.79666 15.313C10.0602 15.2914 10.3224 15.2439 10.5746 15.1154C10.9509 14.9236 11.2569 14.6176 11.4487 14.2413C11.5772 13.9891 11.6247 13.7269 11.6463 13.4634C11.6667 13.2136 11.6667 12.9105 11.6666 12.559V7.44099C11.6667 7.08955 11.6667 6.78637 11.6463 6.53665C11.6247 6.27309 11.5772 6.01095 11.4487 5.75869C11.2569 5.38237 10.9509 5.07641 10.5746 4.88466C10.3224 4.75613 10.0602 4.70859 9.79666 4.68705C9.54695 4.66665 9.24376 4.66666 8.89233 4.66667ZM9.13804 8.80474C9.39839 8.54439 9.39839 8.12228 9.13804 7.86193C8.87769 7.60159 8.45558 7.60159 8.19523 7.86193L5.66664 10.3905L4.80471 9.5286C4.54436 9.26825 4.12225 9.26825 3.8619 9.5286C3.60155 9.78895 3.60155 10.2111 3.8619 10.4714L5.19523 11.8047C5.45558 12.0651 5.87769 12.0651 6.13804 11.8047L9.13804 8.80474Z" fill="#667085" />
-          <path d="M12.8923 0.666672H7.77427C7.42285 0.666661 7.11966 0.666651 6.86995 0.687053C6.60639 0.708587 6.34424 0.756131 6.09199 0.884661C5.71567 1.07641 5.40971 1.38237 5.21796 1.75869C5.08943 2.01095 5.04188 2.27309 5.02035 2.53665C5.00206 2.76051 5.00018 3.02733 4.99999 3.33336L8.92055 3.33335C9.2463 3.33327 9.59951 3.33319 9.90523 3.35816C10.2512 3.38644 10.7084 3.4564 11.1799 3.69667C11.8071 4.01625 12.3171 4.52618 12.6367 5.15339C12.8769 5.62493 12.9469 6.08208 12.9752 6.42809C13.0001 6.73382 13.0001 7.08702 13 7.41279L13 11.3333C13.306 11.3331 13.5728 11.3313 13.7967 11.313C14.0602 11.2914 14.3224 11.2439 14.5746 11.1154C14.9509 10.9236 15.2569 10.6176 15.4487 10.2413C15.5772 9.98907 15.6247 9.72692 15.6463 9.46336C15.6667 9.21366 15.6666 8.91051 15.6666 8.5591V3.44099C15.6666 3.08959 15.6667 2.78635 15.6463 2.53665C15.6247 2.27309 15.5772 2.01095 15.4487 1.75869C15.2569 1.38237 14.9509 1.07641 14.5746 0.884661C14.3224 0.756131 14.0602 0.708587 13.7967 0.687053C13.5469 0.666651 13.2438 0.666661 12.8923 0.666672Z" fill="#667085" />
-        </svg>
-
-      )
+      return selected
+        ? (
+          <svg width="17" height="16" viewBox="0 0 17 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+            <path fillRule="evenodd" clipRule="evenodd" d="M8.89233 4.66669H3.77428C3.42285 4.66668 3.11966 4.66667 2.86995 4.68707C2.60639 4.7086 2.34424 4.75615 2.09199 4.88468C1.71567 5.07642 1.4097 5.38238 1.21796 5.75871C1.08943 6.01096 1.04188 6.27311 1.02035 6.53667C0.999949 6.78638 0.999959 7.08954 0.99997 7.44096V12.5591C0.999959 12.9105 0.999949 13.2137 1.02035 13.4634C1.04188 13.7269 1.08943 13.9891 1.21796 14.2413C1.4097 14.6177 1.71567 14.9236 2.09199 15.1154C2.34424 15.2439 2.60639 15.2914 2.86995 15.313C3.11965 15.3334 3.4228 15.3334 3.77421 15.3334H8.89232C9.24372 15.3334 9.54696 15.3334 9.79666 15.313C10.0602 15.2914 10.3224 15.2439 10.5746 15.1154C10.9509 14.9236 11.2569 14.6177 11.4487 14.2413C11.5772 13.9891 11.6247 13.7269 11.6463 13.4634C11.6667 13.2137 11.6667 12.9105 11.6666 12.559V7.44101C11.6667 7.08957 11.6667 6.78639 11.6463 6.53667C11.6247 6.27311 11.5772 6.01096 11.4487 5.75871C11.2569 5.38238 10.9509 5.07642 10.5746 4.88468C10.3224 4.75615 10.0602 4.7086 9.79666 4.68707C9.54695 4.66667 9.24376 4.66668 8.89233 4.66669ZM9.13804 8.80476C9.39839 8.54441 9.39839 8.1223 9.13804 7.86195C8.87769 7.6016 8.45558 7.6016 8.19523 7.86195L5.66664 10.3905L4.80471 9.52862C4.54436 9.26827 4.12225 9.26827 3.8619 9.52862C3.60155 9.78897 3.60155 10.2111 3.8619 10.4714L5.19523 11.8048C5.45558 12.0651 5.87769 12.0651 6.13804 11.8048L9.13804 8.80476Z" fill="#155EEF" />
+            <path d="M12.8923 0.666688H7.77427C7.42285 0.666676 7.11966 0.666666 6.86995 0.687068C6.60639 0.708602 6.34424 0.756146 6.09199 0.884676C5.71567 1.07642 5.40971 1.38238 5.21796 1.75871C5.08943 2.01096 5.04188 2.27311 5.02035 2.53667C5.00206 2.76053 5.00018 3.02734 4.99999 3.33337L8.92055 3.33336C9.2463 3.33329 9.59951 3.3332 9.90523 3.35818C10.2512 3.38645 10.7084 3.45642 11.1799 3.69668C11.8071 4.01626 12.3171 4.5262 12.6367 5.1534C12.8769 5.62495 12.9469 6.08209 12.9752 6.42811C13.0001 6.73384 13.0001 7.08704 13 7.4128L13 11.3333C13.306 11.3332 13.5728 11.3313 13.7967 11.313C14.0602 11.2914 14.3224 11.2439 14.5746 11.1154C14.9509 10.9236 15.2569 10.6177 15.4487 10.2413C15.5772 9.98908 15.6247 9.72694 15.6463 9.46338C15.6667 9.21368 15.6666 8.91052 15.6666 8.55912V3.44101C15.6666 3.0896 15.6667 2.78637 15.6463 2.53667C15.6247 2.27311 15.5772 2.01096 15.4487 1.75871C15.2569 1.38238 14.9509 1.07642 14.5746 0.884676C14.3224 0.756146 14.0602 0.708602 13.7967 0.687068C13.5469 0.666666 13.2438 0.666676 12.8923 0.666688Z" fill="#155EEF" />
+          </svg>
+        )
+        : (
+          <svg width="17" height="16" viewBox="0 0 17 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+            <path fillRule="evenodd" clipRule="evenodd" d="M8.89233 4.66667H3.77428C3.42285 4.66666 3.11966 4.66665 2.86995 4.68705C2.60639 4.70859 2.34424 4.75613 2.09199 4.88466C1.71567 5.07641 1.4097 5.38237 1.21796 5.75869C1.08943 6.01095 1.04188 6.27309 1.02035 6.53665C0.999949 6.78636 0.999959 7.08953 0.99997 7.44095V12.559C0.999959 12.9105 0.999949 13.2137 1.02035 13.4634C1.04188 13.7269 1.08943 13.9891 1.21796 14.2413C1.4097 14.6176 1.71567 14.9236 2.09199 15.1154C2.34424 15.2439 2.60639 15.2914 2.86995 15.313C3.11965 15.3334 3.4228 15.3334 3.77421 15.3333H8.89232C9.24372 15.3334 9.54696 15.3334 9.79666 15.313C10.0602 15.2914 10.3224 15.2439 10.5746 15.1154C10.9509 14.9236 11.2569 14.6176 11.4487 14.2413C11.5772 13.9891 11.6247 13.7269 11.6463 13.4634C11.6667 13.2136 11.6667 12.9105 11.6666 12.559V7.44099C11.6667 7.08955 11.6667 6.78637 11.6463 6.53665C11.6247 6.27309 11.5772 6.01095 11.4487 5.75869C11.2569 5.38237 10.9509 5.07641 10.5746 4.88466C10.3224 4.75613 10.0602 4.70859 9.79666 4.68705C9.54695 4.66665 9.24376 4.66666 8.89233 4.66667ZM9.13804 8.80474C9.39839 8.54439 9.39839 8.12228 9.13804 7.86193C8.87769 7.60159 8.45558 7.60159 8.19523 7.86193L5.66664 10.3905L4.80471 9.5286C4.54436 9.26825 4.12225 9.26825 3.8619 9.5286C3.60155 9.78895 3.60155 10.2111 3.8619 10.4714L5.19523 11.8047C5.45558 12.0651 5.87769 12.0651 6.13804 11.8047L9.13804 8.80474Z" fill="#667085" />
+            <path d="M12.8923 0.666672H7.77427C7.42285 0.666661 7.11966 0.666651 6.86995 0.687053C6.60639 0.708587 6.34424 0.756131 6.09199 0.884661C5.71567 1.07641 5.40971 1.38237 5.21796 1.75869C5.08943 2.01095 5.04188 2.27309 5.02035 2.53665C5.00206 2.76051 5.00018 3.02733 4.99999 3.33336L8.92055 3.33335C9.2463 3.33327 9.59951 3.33319 9.90523 3.35816C10.2512 3.38644 10.7084 3.4564 11.1799 3.69667C11.8071 4.01625 12.3171 4.52618 12.6367 5.15339C12.8769 5.62493 12.9469 6.08208 12.9752 6.42809C13.0001 6.73382 13.0001 7.08702 13 7.41279L13 11.3333C13.306 11.3331 13.5728 11.3313 13.7967 11.313C14.0602 11.2914 14.3224 11.2439 14.5746 11.1154C14.9509 10.9236 15.2569 10.6176 15.4487 10.2413C15.5772 9.98907 15.6247 9.72692 15.6463 9.46336C15.6667 9.21366 15.6666 8.91051 15.6666 8.5591V3.44099C15.6666 3.08959 15.6667 2.78635 15.6463 2.53665C15.6247 2.27309 15.5772 2.01095 15.4487 1.75869C15.2569 1.38237 14.9509 1.07641 14.5746 0.884661C14.3224 0.756131 14.0602 0.708587 13.7967 0.687053C13.5469 0.666651 13.2438 0.666661 12.8923 0.666672Z" fill="#667085" />
+          </svg>
+        )
+    case 'paragraph':
+      return selected
+        ? (
+          <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+            <g id="align-left">
+              <g id="Solid">
+                <path fillRule="evenodd" clipRule="evenodd" d="M1.33334 6.66665C1.33334 6.29846 1.63182 5.99998 2.00001 5.99998H10.6667C11.0349 5.99998 11.3333 6.29846 11.3333 6.66665C11.3333 7.03484 11.0349 7.33331 10.6667 7.33331H2.00001C1.63182 7.33331 1.33334 7.03484 1.33334 6.66665Z" fill="#155EEF"/>
+                <path fillRule="evenodd" clipRule="evenodd" d="M1.33334 3.99998C1.33334 3.63179 1.63182 3.33331 2.00001 3.33331H13.3333C13.7015 3.33331 14 3.63179 14 3.99998C14 4.36817 13.7015 4.66665 13.3333 4.66665H2.00001C1.63182 4.66665 1.33334 4.36817 1.33334 3.99998Z" fill="#155EEF"/>
+                <path fillRule="evenodd" clipRule="evenodd" d="M1.33334 9.33331C1.33334 8.96512 1.63182 8.66665 2.00001 8.66665H13.3333C13.7015 8.66665 14 8.96512 14 9.33331C14 9.7015 13.7015 9.99998 13.3333 9.99998H2.00001C1.63182 9.99998 1.33334 9.7015 1.33334 9.33331Z" fill="#155EEF"/>
+                <path fillRule="evenodd" clipRule="evenodd" d="M1.33334 12C1.33334 11.6318 1.63182 11.3333 2.00001 11.3333H10.6667C11.0349 11.3333 11.3333 11.6318 11.3333 12C11.3333 12.3682 11.0349 12.6666 10.6667 12.6666H2.00001C1.63182 12.6666 1.33334 12.3682 1.33334 12Z" fill="#155EEF"/>
+              </g>
+            </g>
+          </svg>
+        )
+        : (
+          <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+            <g id="align-left">
+              <g id="Solid">
+                <path fillRule="evenodd" clipRule="evenodd" d="M1.33334 6.66666C1.33334 6.29847 1.63182 5.99999 2.00001 5.99999H10.6667C11.0349 5.99999 11.3333 6.29847 11.3333 6.66666C11.3333 7.03485 11.0349 7.33333 10.6667 7.33333H2.00001C1.63182 7.33333 1.33334 7.03485 1.33334 6.66666Z" fill="#667085"/>
+                <path fillRule="evenodd" clipRule="evenodd" d="M1.33334 3.99999C1.33334 3.63181 1.63182 3.33333 2.00001 3.33333H13.3333C13.7015 3.33333 14 3.63181 14 3.99999C14 4.36818 13.7015 4.66666 13.3333 4.66666H2.00001C1.63182 4.66666 1.33334 4.36818 1.33334 3.99999Z" fill="#667085"/>
+                <path fillRule="evenodd" clipRule="evenodd" d="M1.33334 9.33333C1.33334 8.96514 1.63182 8.66666 2.00001 8.66666H13.3333C13.7015 8.66666 14 8.96514 14 9.33333C14 9.70152 13.7015 10 13.3333 10H2.00001C1.63182 10 1.33334 9.70152 1.33334 9.33333Z" fill="#667085"/>
+                <path fillRule="evenodd" clipRule="evenodd" d="M1.33334 12C1.33334 11.6318 1.63182 11.3333 2.00001 11.3333H10.6667C11.0349 11.3333 11.3333 11.6318 11.3333 12C11.3333 12.3682 11.0349 12.6667 10.6667 12.6667H2.00001C1.63182 12.6667 1.33334 12.3682 1.33334 12Z" fill="#667085"/>
+              </g>
+            </g>
+          </svg>
+        )
     case 'string':
     default:
-      return selected ? (
-        <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-          <path fillRule="evenodd" clipRule="evenodd" d="M5.17246 1.33333H10.8275C11.3642 1.33332 11.8071 1.33331 12.1679 1.36279C12.5426 1.39341 12.8871 1.45912 13.2106 1.62398C13.7124 1.87964 14.1203 2.28759 14.376 2.78935C14.5409 3.11291 14.6066 3.45738 14.6372 3.83211C14.6667 4.19291 14.6667 4.63581 14.6667 5.17245V10.8275C14.6667 11.3642 14.6667 11.8071 14.6372 12.1679C14.6066 12.5426 14.5409 12.8871 14.376 13.2106C14.1203 13.7124 13.7124 14.1203 13.2106 14.376C12.8871 14.5409 12.5426 14.6066 12.1679 14.6372C11.8071 14.6667 11.3642 14.6667 10.8275 14.6667H5.17245C4.63581 14.6667 4.1929 14.6667 3.83211 14.6372C3.45738 14.6066 3.11291 14.5409 2.78935 14.376C2.28759 14.1203 1.87964 13.7124 1.62398 13.2106C1.45912 12.8871 1.39341 12.5426 1.36279 12.1679C1.33331 11.8071 1.33332 11.3642 1.33333 10.8275V5.17245C1.33332 4.63581 1.33331 4.1929 1.36279 3.83211C1.39341 3.45738 1.45912 3.11291 1.62398 2.78935C1.87964 2.28759 2.28759 1.87964 2.78935 1.62398C3.11291 1.45912 3.45738 1.39341 3.83211 1.36279C4.1929 1.33331 4.63583 1.33332 5.17246 1.33333ZM4.66666 4.66666C4.66666 4.29847 4.96514 3.99999 5.33333 3.99999H10.6667C11.0349 3.99999 11.3333 4.29847 11.3333 4.66666C11.3333 5.03485 11.0349 5.33333 10.6667 5.33333H8.66666V11.3333C8.66666 11.7015 8.36818 12 7.99999 12C7.6318 12 7.33333 11.7015 7.33333 11.3333V5.33333H5.33333C4.96514 5.33333 4.66666 5.03485 4.66666 4.66666Z" fill="#155EEF" />
-        </svg>
-
-      ) : (<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-        <path fillRule="evenodd" clipRule="evenodd" d="M5.17246 1.33331H10.8275C11.3642 1.33331 11.8071 1.3333 12.1679 1.36278C12.5426 1.39339 12.8871 1.4591 13.2106 1.62396C13.7124 1.87963 14.1203 2.28757 14.376 2.78934C14.5409 3.1129 14.6066 3.45737 14.6372 3.8321C14.6667 4.19289 14.6667 4.63579 14.6667 5.17243V10.8275C14.6667 11.3642 14.6667 11.8071 14.6372 12.1679C14.6066 12.5426 14.5409 12.8871 14.376 13.2106C14.1203 13.7124 13.7124 14.1203 13.2106 14.376C12.8871 14.5409 12.5426 14.6066 12.1679 14.6372C11.8071 14.6667 11.3642 14.6667 10.8275 14.6666H5.17245C4.63581 14.6667 4.1929 14.6667 3.83211 14.6372C3.45738 14.6066 3.11291 14.5409 2.78935 14.376C2.28759 14.1203 1.87964 13.7124 1.62398 13.2106C1.45912 12.8871 1.39341 12.5426 1.36279 12.1679C1.33331 11.8071 1.33332 11.3642 1.33333 10.8275V5.17244C1.33332 4.6358 1.33331 4.19289 1.36279 3.8321C1.39341 3.45737 1.45912 3.1129 1.62398 2.78934C1.87964 2.28757 2.28759 1.87963 2.78935 1.62396C3.11291 1.4591 3.45738 1.39339 3.83211 1.36278C4.1929 1.3333 4.63583 1.33331 5.17246 1.33331ZM4.66666 4.66665C4.66666 4.29846 4.96514 3.99998 5.33333 3.99998H10.6667C11.0349 3.99998 11.3333 4.29846 11.3333 4.66665C11.3333 5.03484 11.0349 5.33331 10.6667 5.33331H8.66666V11.3333C8.66666 11.7015 8.36818 12 7.99999 12C7.6318 12 7.33333 11.7015 7.33333 11.3333V5.33331H5.33333C4.96514 5.33331 4.66666 5.03484 4.66666 4.66665Z" fill="#667085" />
-      </svg>)
+      return selected
+        ? (
+          <svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+            <path fillRule="evenodd" clipRule="evenodd" d="M5.17246 1.33333H10.8275C11.3642 1.33332 11.8071 1.33331 12.1679 1.36279C12.5426 1.39341 12.8871 1.45912 13.2106 1.62398C13.7124 1.87964 14.1203 2.28759 14.376 2.78935C14.5409 3.11291 14.6066 3.45738 14.6372 3.83211C14.6667 4.19291 14.6667 4.63581 14.6667 5.17245V10.8275C14.6667 11.3642 14.6667 11.8071 14.6372 12.1679C14.6066 12.5426 14.5409 12.8871 14.376 13.2106C14.1203 13.7124 13.7124 14.1203 13.2106 14.376C12.8871 14.5409 12.5426 14.6066 12.1679 14.6372C11.8071 14.6667 11.3642 14.6667 10.8275 14.6667H5.17245C4.63581 14.6667 4.1929 14.6667 3.83211 14.6372C3.45738 14.6066 3.11291 14.5409 2.78935 14.376C2.28759 14.1203 1.87964 13.7124 1.62398 13.2106C1.45912 12.8871 1.39341 12.5426 1.36279 12.1679C1.33331 11.8071 1.33332 11.3642 1.33333 10.8275V5.17245C1.33332 4.63581 1.33331 4.1929 1.36279 3.83211C1.39341 3.45738 1.45912 3.11291 1.62398 2.78935C1.87964 2.28759 2.28759 1.87964 2.78935 1.62398C3.11291 1.45912 3.45738 1.39341 3.83211 1.36279C4.1929 1.33331 4.63583 1.33332 5.17246 1.33333ZM4.66666 4.66666C4.66666 4.29847 4.96514 3.99999 5.33333 3.99999H10.6667C11.0349 3.99999 11.3333 4.29847 11.3333 4.66666C11.3333 5.03485 11.0349 5.33333 10.6667 5.33333H8.66666V11.3333C8.66666 11.7015 8.36818 12 7.99999 12C7.6318 12 7.33333 11.7015 7.33333 11.3333V5.33333H5.33333C4.96514 5.33333 4.66666 5.03485 4.66666 4.66666Z" fill="#155EEF" />
+          </svg>
+        )
+        : (<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
+          <path fillRule="evenodd" clipRule="evenodd" d="M5.17246 1.33331H10.8275C11.3642 1.33331 11.8071 1.3333 12.1679 1.36278C12.5426 1.39339 12.8871 1.4591 13.2106 1.62396C13.7124 1.87963 14.1203 2.28757 14.376 2.78934C14.5409 3.1129 14.6066 3.45737 14.6372 3.8321C14.6667 4.19289 14.6667 4.63579 14.6667 5.17243V10.8275C14.6667 11.3642 14.6667 11.8071 14.6372 12.1679C14.6066 12.5426 14.5409 12.8871 14.376 13.2106C14.1203 13.7124 13.7124 14.1203 13.2106 14.376C12.8871 14.5409 12.5426 14.6066 12.1679 14.6372C11.8071 14.6667 11.3642 14.6667 10.8275 14.6666H5.17245C4.63581 14.6667 4.1929 14.6667 3.83211 14.6372C3.45738 14.6066 3.11291 14.5409 2.78935 14.376C2.28759 14.1203 1.87964 13.7124 1.62398 13.2106C1.45912 12.8871 1.39341 12.5426 1.36279 12.1679C1.33331 11.8071 1.33332 11.3642 1.33333 10.8275V5.17244C1.33332 4.6358 1.33331 4.19289 1.36279 3.8321C1.39341 3.45737 1.45912 3.1129 1.62398 2.78934C1.87964 2.28757 2.28759 1.87963 2.78935 1.62396C3.11291 1.4591 3.45738 1.39339 3.83211 1.36278C4.1929 1.3333 4.63583 1.33331 5.17246 1.33331ZM4.66666 4.66665C4.66666 4.29846 4.96514 3.99998 5.33333 3.99998H10.6667C11.0349 3.99998 11.3333 4.29846 11.3333 4.66665C11.3333 5.03484 11.0349 5.33331 10.6667 5.33331H8.66666V11.3333C8.66666 11.7015 8.36818 12 7.99999 12C7.6318 12 7.33333 11.7015 7.33333 11.3333V5.33331H5.33333C4.96514 5.33331 4.66666 5.03484 4.66666 4.66665Z" fill="#667085" />
+        </svg>)
   }
 }
 
 const SelectTypeItem: FC<ISelectTypeItemProps> = ({
   type,
   selected,
-  onClick
+  onClick,
 }) => {
   const { t } = useTranslation()
   const typeName = t(`appDebug.variableConig.${type}`)
 
   return (
     <div
-      className={cn(s.item, selected && s.selected, 'space-x-2')}
+      className={cn(s.item, selected && s.selected, 'space-y-1')}
       onClick={onClick}
     >
-      <Icon type={type} selected={selected} />
+      <div className='shrink-0'>
+        <Icon type={type} selected={selected} />
+      </div>
       <span className={cn(s.text)}>{typeName}</span>
     </div>
   )

web/app/components/app/configuration/config-var/select-type-item/style.module.css

@@ -1,9 +1,10 @@
 .item {
   display: flex;
+  flex-direction: column;
+  justify-content: center;
   align-items: center;
-  height: 32px;
-  width: 133px;
-  padding-left: 12px;
+  height: 58px;
+  width: 98px;
   border-radius: 8px;
   border: 1px solid #EAECF0;
   box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05);

web/app/components/app/configuration/config/feature/choose-feature/feature-item/style.module.css

@@ -26,4 +26,8 @@
 
 .speechToTextPreview {
   background-image: url(./preview-imgs/speech-to-text.svg);
+}
+
+.citationPreview {
+  background-image: url(./preview-imgs/citation.svg);
 }

web/app/components/app/configuration/config/feature/choose-feature/index.tsx

@@ -8,11 +8,13 @@ import FeatureItem from './feature-item'
 import Modal from '@/app/components/base/modal'
 import SuggestedQuestionsAfterAnswerIcon from '@/app/components/app/configuration/base/icons/suggested-questions-after-answer-icon'
 import { Microphone01 } from '@/app/components/base/icons/src/vender/solid/mediaAndDevices'
+import { Citations } from '@/app/components/base/icons/src/vender/solid/editor'
 type IConfig = {
   openingStatement: boolean
   moreLikeThis: boolean
   suggestedQuestionsAfterAnswer: boolean
   speechToText: boolean
+  citation: boolean
 }
 
 export type IChooseFeatureProps = {
@@ -85,6 +87,14 @@ const ChooseFeature: FC<IChooseFeatureProps> = ({
                   />
                 )
               }
+              <FeatureItem
+                icon={<Citations className='w-4 h-4 text-[#FD853A]' />}
+                previewImgClassName='citationPreview'
+                title={t('appDebug.feature.citation.title')}
+                description={t('appDebug.feature.citation.description')}
+                value={config.citation}
+                onChange={value => onChange('citation', value)}
+              />
             </>
           </FeatureGroup>
         )}

web/app/components/app/configuration/config/feature/use-feature.tsx

@@ -9,6 +9,8 @@ function useFeature({
   setSuggestedQuestionsAfterAnswer,
   speechToText,
   setSpeechToText,
+  citation,
+  setCitation,
 }: {
   introduction: string
   setIntroduction: (introduction: string) => void
@@ -18,6 +20,8 @@ function useFeature({
   setSuggestedQuestionsAfterAnswer: (suggestedQuestionsAfterAnswer: boolean) => void
   speechToText: boolean
   setSpeechToText: (speechToText: boolean) => void
+  citation: boolean
+  setCitation: (citation: boolean) => void
 }) {
   const [tempshowOpeningStatement, setTempShowOpeningStatement] = React.useState(!!introduction)
   useEffect(() => {
@@ -36,6 +40,7 @@ function useFeature({
     moreLikeThis,
     suggestedQuestionsAfterAnswer,
     speechToText,
+    citation,
   }
   const handleFeatureChange = (key: string, value: boolean) => {
     switch (key) {
@@ -53,6 +58,9 @@ function useFeature({
         break
       case 'speechToText':
         setSpeechToText(value)
+        break
+      case 'citation':
+        setCitation(value)
     }
   }
   return {

web/app/components/app/configuration/config/index.tsx

@@ -36,6 +36,8 @@ const Config: FC = () => {
     setSuggestedQuestionsAfterAnswerConfig,
     speechToTextConfig,
     setSpeechToTextConfig,
+    citationConfig,
+    setCitationConfig,
   } = useContext(ConfigContext)
   const isChatApp = mode === AppType.chat
   const { speech2textDefaultModel } = useProviderContext()
@@ -88,9 +90,15 @@ const Config: FC = () => {
         draft.enabled = value
       }))
     },
+    citation: citationConfig.enabled,
+    setCitation: (value) => {
+      setCitationConfig(produce(citationConfig, (draft) => {
+        draft.enabled = value
+      }))
+    },
   })
 
-  const hasChatConfig = isChatApp && (featureConfig.openingStatement || featureConfig.suggestedQuestionsAfterAnswer || (featureConfig.speechToText && !!speech2textDefaultModel))
+  const hasChatConfig = isChatApp && (featureConfig.openingStatement || featureConfig.suggestedQuestionsAfterAnswer || (featureConfig.speechToText && !!speech2textDefaultModel) || featureConfig.citation)
   const hasToolbox = false
 
   const [showAutomatic, { setTrue: showAutomaticTrue, setFalse: showAutomaticFalse }] = useBoolean(false)
@@ -161,6 +169,7 @@ const Config: FC = () => {
               }
               isShowSuggestedQuestionsAfterAnswer={featureConfig.suggestedQuestionsAfterAnswer}
               isShowSpeechText={featureConfig.speechToText && !!speech2textDefaultModel}
+              isShowCitation={featureConfig.citation}
             />
           )
         }

web/app/components/app/configuration/dataset-config/card-item/index.tsx

@@ -29,9 +29,9 @@ const CardItem: FC<ICardItemProps> = ({
     <div
       className={
         cn(className, s.card,
-          'flex items-center justify-between rounded-xl  px-3 py-2.5 bg-white border border-gray-200  cursor-pointer')
+          'relative flex items-center rounded-xl  px-3 py-2.5 bg-white border border-gray-200  cursor-pointer')
       }>
-      <div className='shrink-0 flex items-center space-x-2'>
+      <div className='flex items-center space-x-2'>
         <div className={cn(!config.embedding_available && 'opacity-50')}>
           <TypeIcon type="upload_file" />
         </div>
@@ -47,13 +47,13 @@ const CardItem: FC<ICardItemProps> = ({
               </Tooltip>
             )}
           </div>
-          <div className={cn('flex text-xs text-gray-500', !config.embedding_available && 'opacity-50')}>
+          <div className={cn('max-w-[150px] flex text-xs text-gray-500', !config.embedding_available && 'opacity-50')}>
             {formatNumber(config.word_count)} {t('appDebug.feature.dataSet.words')} · {formatNumber(config.document_count)} {t('appDebug.feature.dataSet.textBlocks')}
           </div>
         </div>
       </div>
 
-      {!readonly && <RemoveIcon className={`${s.deleteBtn} shrink-0`} onClick={() => onRemove(config.id)} />}
+      {!readonly && <RemoveIcon className={`${s.deleteBtn} absolute right-1 top-1/2 translate-y-[-50%]`} onClick={() => onRemove(config.id)} />}
     </div>
   )
 }

web/app/components/app/configuration/dataset-config/card-item/style.module.css

@@ -8,9 +8,9 @@
 }
 
 .deleteBtn {
-  display: none;
+  visibility: hidden;
 }
 
 .card:hover .deleteBtn {
-  display: block;
+  visibility: visible;
 }

web/app/components/app/configuration/debug/index.tsx

@@ -40,6 +40,7 @@ const Debug: FC<IDebug> = ({
     introduction,
     suggestedQuestionsAfterAnswerConfig,
     speechToTextConfig,
+    citationConfig,
     moreLikeThisConfig,
     inputs,
     // setInputs,
@@ -162,6 +163,7 @@ const Debug: FC<IDebug> = ({
       },
       suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
       speech_to_text: speechToTextConfig,
+      retriever_resource: citationConfig,
       agent_mode: {
         enabled: true,
         tools: [...postDatasets],
@@ -199,7 +201,7 @@ const Debug: FC<IDebug> = ({
     setChatList(newList)
 
     // answer
-    const responseItem = {
+    const responseItem: IChatItem = {
       id: `${Date.now()}`,
       content: '',
       isAnswer: true,
@@ -266,6 +268,19 @@ const Debug: FC<IDebug> = ({
           setIsShowSuggestion(true)
         }
       },
+      onMessageEnd: (messageEnd) => {
+        responseItem.citation = messageEnd.retriever_resources
+
+        const newListWithAnswer = produce(
+          getChatList().filter(item => item.id !== responseItem.id && item.id !== placeholderAnswerId),
+          (draft) => {
+            if (!draft.find(item => item.id === questionId))
+              draft.push({ ...questionItem })
+
+            draft.push({ ...responseItem })
+          })
+        setChatList(newListWithAnswer)
+      },
       onError() {
         setResponsingFalse()
         // role back placeholder answer
@@ -282,7 +297,6 @@ const Debug: FC<IDebug> = ({
       setChatList([])
   }, [controlClearChatMessage])
 
-  const [completionQuery, setCompletionQuery] = useState('')
   const [completionRes, setCompletionRes] = useState('')
 
   const sendTextCompletion = async () => {
@@ -294,11 +308,6 @@ const Debug: FC<IDebug> = ({
     if (!checkCanSend())
       return
 
-    if (!completionQuery) {
-      logError(t('appDebug.errorMessage.queryRequired'))
-      return false
-    }
-
     const postDatasets = dataSets.map(({ id }) => ({
       dataset: {
         enabled: true,
@@ -312,6 +321,7 @@ const Debug: FC<IDebug> = ({
       opening_statement: introduction,
       suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
       speech_to_text: speechToTextConfig,
+      retriever_resource: citationConfig,
       more_like_this: moreLikeThisConfig,
       agent_mode: {
         enabled: true,
@@ -326,7 +336,6 @@ const Debug: FC<IDebug> = ({
 
     const data = {
       inputs,
-      query: completionQuery,
       model_config: postModelConfig,
     }
 
@@ -364,8 +373,6 @@ const Debug: FC<IDebug> = ({
         </div>
         <PromptValuePanel
           appType={mode as AppType}
-          value={completionQuery}
-          onChange={setCompletionQuery}
           onSend={sendTextCompletion}
         />
       </div>
@@ -391,6 +398,8 @@ const Debug: FC<IDebug> = ({
                   isShowSuggestion={doShowSuggestion}
                   suggestionList={suggestQuestions}
                   isShowSpeechToText={speechToTextConfig.enabled && !!speech2textDefaultModel}
+                  isShowCitation={citationConfig.enabled}
+                  isShowCitationHitInfo
                 />
               </div>
             </div>

web/app/components/app/configuration/features/chat-group/citation/index.tsx

@@ -0,0 +1,25 @@
+'use client'
+import React, { type FC } from 'react'
+import { useTranslation } from 'react-i18next'
+import Panel from '@/app/components/app/configuration/base/feature-panel'
+import { Citations } from '@/app/components/base/icons/src/vender/solid/editor'
+
+const Citation: FC = () => {
+  const { t } = useTranslation()
+
+  return (
+    <Panel
+      title={
+        <div className='flex items-center gap-2'>
+          <div>{t('appDebug.feature.citation.title')}</div>
+        </div>
+      }
+      headerIcon={<Citations className='w-4 h-4 text-[#FD853A]' />}
+      headerRight={
+        <div className='text-xs text-gray-500'>{t('appDebug.feature.citation.resDes')}</div>
+      }
+      noBodySpacing
+    />
+  )
+}
+export default React.memo(Citation)

web/app/components/app/configuration/features/chat-group/index.tsx

@@ -7,6 +7,7 @@ import type { IOpeningStatementProps } from './opening-statement'
 import OpeningStatement from './opening-statement'
 import SuggestedQuestionsAfterAnswer from './suggested-questions-after-answer'
 import SpeechToText from './speech-to-text'
+import Citation from './citation'
 
 /*
 * Include
@@ -19,12 +20,14 @@ type ChatGroupProps = {
   openingStatementConfig: IOpeningStatementProps
   isShowSuggestedQuestionsAfterAnswer: boolean
   isShowSpeechText: boolean
+  isShowCitation: boolean
 }
 const ChatGroup: FC<ChatGroupProps> = ({
   isShowOpeningStatement,
   openingStatementConfig,
   isShowSuggestedQuestionsAfterAnswer,
   isShowSpeechText,
+  isShowCitation,
 }) => {
   const { t } = useTranslation()
 
@@ -43,6 +46,11 @@ const ChatGroup: FC<ChatGroupProps> = ({
             <SpeechToText />
           )
         }
+        {
+          isShowCitation && (
+            <Citation />
+          )
+        }
       </div>
     </div>
   )

web/app/components/app/configuration/index.tsx

@@ -6,6 +6,7 @@ import { useContext } from 'use-context-selector'
 import { usePathname } from 'next/navigation'
 import produce from 'immer'
 import { useBoolean } from 'ahooks'
+import cn from 'classnames'
 import Button from '../../base/button'
 import Loading from '../../base/loading'
 import type { CompletionParams, Inputs, ModelConfig, MoreLikeThisConfig, PromptConfig, PromptVariable } from '@/models/debug'
@@ -24,6 +25,7 @@ import { promptVariablesToUserInputsForm, userInputsFormToPromptVariables } from
 import { fetchDatasets } from '@/service/datasets'
 import AccountSetting from '@/app/components/header/account-setting'
 import { useProviderContext } from '@/context/provider-context'
+import { AppType } from '@/types/app'
 
 const Configuration: FC = () => {
   const { t } = useTranslation()
@@ -56,6 +58,9 @@ const Configuration: FC = () => {
   const [speechToTextConfig, setSpeechToTextConfig] = useState<MoreLikeThisConfig>({
     enabled: false,
   })
+  const [citationConfig, setCitationConfig] = useState<MoreLikeThisConfig>({
+    enabled: false,
+  })
   const [formattingChanged, setFormattingChanged] = useState(false)
   const [inputs, setInputs] = useState<Inputs>({})
   const [query, setQuery] = useState('')
@@ -77,6 +82,7 @@ const Configuration: FC = () => {
     more_like_this: null,
     suggested_questions_after_answer: null,
     speech_to_text: null,
+    retriever_resource: null,
     dataSets: [],
   })
 
@@ -110,6 +116,9 @@ const Configuration: FC = () => {
     setSpeechToTextConfig(modelConfig.speech_to_text || {
       enabled: false,
     })
+    setCitationConfig(modelConfig.retriever_resource || {
+      enabled: false,
+    })
   }
 
   const { textGenerationModelList } = useProviderContext()
@@ -159,6 +168,9 @@ const Configuration: FC = () => {
       if (modelConfig.speech_to_text)
         setSpeechToTextConfig(modelConfig.speech_to_text)
 
+      if (modelConfig.retriever_resource)
+        setCitationConfig(modelConfig.retriever_resource)
+
       const config = {
         modelConfig: {
           provider: model.provider,
@@ -171,6 +183,7 @@ const Configuration: FC = () => {
           more_like_this: modelConfig.more_like_this,
           suggested_questions_after_answer: modelConfig.suggested_questions_after_answer,
           speech_to_text: modelConfig.speech_to_text,
+          retriever_resource: modelConfig.retriever_resource,
           dataSets: datasets || [],
         },
         completionParams: model.completion_params,
@@ -182,11 +195,16 @@ const Configuration: FC = () => {
     })
   }, [appId])
 
+  const cannotPublish = mode === AppType.completion && !modelConfig.configs.prompt_template
   const saveAppConfig = async () => {
     const modelId = modelConfig.model_id
     const promptTemplate = modelConfig.configs.prompt_template
     const promptVariables = modelConfig.configs.prompt_variables
 
+    if (cannotPublish) {
+      notify({ type: 'error', message: t('appDebug.otherError.promptNoBeEmpty'), duration: 3000 })
+      return
+    }
     const postDatasets = dataSets.map(({ id }) => ({
       dataset: {
         enabled: true,
@@ -202,6 +220,7 @@ const Configuration: FC = () => {
       more_like_this: moreLikeThisConfig,
       suggested_questions_after_answer: suggestedQuestionsAfterAnswerConfig,
       speech_to_text: speechToTextConfig,
+      retriever_resource: citationConfig,
       agent_mode: {
         enabled: true,
         tools: [...postDatasets],
@@ -219,6 +238,7 @@ const Configuration: FC = () => {
       draft.more_like_this = moreLikeThisConfig
       draft.suggested_questions_after_answer = suggestedQuestionsAfterAnswerConfig
       draft.speech_to_text = speechToTextConfig
+      draft.retriever_resource = citationConfig
       draft.dataSets = dataSets
     })
     setPublishedConfig({
@@ -263,6 +283,8 @@ const Configuration: FC = () => {
       setSuggestedQuestionsAfterAnswerConfig,
       speechToTextConfig,
       setSpeechToTextConfig,
+      citationConfig,
+      setCitationConfig,
       formattingChanged,
       setFormattingChanged,
       inputs,
@@ -296,7 +318,7 @@ const Configuration: FC = () => {
               />
               <div className='mx-3 w-[1px] h-[14px] bg-gray-200'></div>
               <Button onClick={() => setShowConfirm(true)} className='shrink-0 mr-2 w-[70px] !h-8 !text-[13px] font-medium'>{t('appDebug.operation.resetConfig')}</Button>
-              <Button type='primary' onClick={saveAppConfig} className='shrink-0 w-[70px] !h-8 !text-[13px] font-medium'>{t('appDebug.operation.applyConfig')}</Button>
+              <Button type='primary' onClick={saveAppConfig} className={cn(cannotPublish && '!bg-primary-200 !cursor-not-allowed', 'shrink-0 w-[70px] !h-8 !text-[13px] font-medium')}>{t('appDebug.operation.applyConfig')}</Button>
             </div>
           </div>
           <div className='flex grow h-[200px]'>

web/app/components/app/configuration/prompt-value-panel/index.tsx

@@ -14,11 +14,10 @@ import Select from '@/app/components/base/select'
 import { DEFAULT_VALUE_MAX_LEN } from '@/config'
 import Button from '@/app/components/base/button'
 import { ChevronDown, ChevronRight } from '@/app/components/base/icons/src/vender/line/arrows'
+import Tooltip from '@/app/components/base/tooltip-plus'
 
 export type IPromptValuePanelProps = {
   appType: AppType
-  value?: string
-  onChange?: (value: string) => void
   onSend?: () => void
 }
 
@@ -32,12 +31,10 @@ const starIcon = (
 
 const PromptValuePanel: FC<IPromptValuePanelProps> = ({
   appType,
-  value,
-  onChange,
   onSend,
 }) => {
   const { t } = useTranslation()
-  const { modelConfig, inputs, setInputs } = useContext(ConfigContext)
+  const { modelConfig, inputs, setInputs, mode } = useContext(ConfigContext)
   const [promptPreviewCollapse, setPromptPreviewCollapse] = useState(false)
   const [userInputFieldCollapse, setUserInputFieldCollapse] = useState(false)
   const promptTemplate = modelConfig.configs.prompt_template
@@ -53,6 +50,19 @@ const PromptValuePanel: FC<IPromptValuePanelProps> = ({
     return obj
   })()
 
+  const canNotRun = mode === AppType.completion && !modelConfig.configs.prompt_template
+  const renderRunButton = () => {
+    return (
+      <Button
+        type="primary"
+        disabled={canNotRun}
+        onClick={() => onSend && onSend()}
+        className="w-[80px] !h-8">
+        <PlayIcon className="shrink-0 w-4 h-4 mr-1" aria-hidden="true" />
+        <span className='uppercase text-[13px]'>{t('appDebug.inputs.run')}</span>
+      </Button>
+    )
+  }
   const handleInputValueChange = (key: string, value: string) => {
     if (!(key in promptVariableObj))
       return
@@ -65,6 +75,14 @@ const PromptValuePanel: FC<IPromptValuePanelProps> = ({
     setInputs(newInputs)
   }
 
+  const onClear = () => {
+    const newInputs: Record<string, any> = {}
+    promptVariables.forEach((item) => {
+      newInputs[item.key] = ''
+    })
+    setInputs(newInputs)
+  }
+
   const promptPreview = (
     <div className='py-3 rounded-t-xl bg-indigo-25'>
       <div className="px-4">
@@ -125,83 +143,78 @@ const PromptValuePanel: FC<IPromptValuePanelProps> = ({
             <div className="mt-1 text-xs leading-normal text-gray-500">{t('appDebug.inputs.completionVarTip')}</div>
           )}
         </div>
-        {
-          !userInputFieldCollapse && (
-            <>
-              {
-                promptVariables.length > 0
-                  ? (
-                    <div className="space-y-3 ">
-                      {promptVariables.map(({ key, name, type, options, max_length, required }) => (
-                        <div key={key} className="flex items-center justify-between">
-                          <div className="mr-1 shrink-0 w-[120px] text-sm text-gray-900">{name || key}</div>
-                          {type === 'select'
-                            ? (
-                              <Select
-                                className='w-full'
-                                defaultValue={inputs[key] as string}
-                                onSelect={(i) => { handleInputValueChange(key, i.value as string) }}
-                                items={(options || []).map(i => ({ name: i, value: i }))}
-                                allowSearch={false}
-                                bgClassName='bg-gray-50'
-                                overlayClassName='z-[11]'
-                              />
-                            )
-                            : (
-                              <input
-                                className="w-full px-3 text-sm leading-9 text-gray-900 border-0 rounded-lg grow h-9 bg-gray-50 focus:outline-none focus:ring-1 focus:ring-inset focus:ring-gray-200"
-                                placeholder={`${name}${!required ? `(${t('appDebug.variableTable.optional')})` : ''}`}
-                                type="text"
-                                value={inputs[key] ? `${inputs[key]}` : ''}
-                                onChange={(e) => { handleInputValueChange(key, e.target.value) }}
-                                maxLength={max_length || DEFAULT_VALUE_MAX_LEN}
-                              />
-                            )}
+        {!userInputFieldCollapse && (
+          <>
+            {
+              promptVariables.length > 0
+                ? (
+                  <div className="space-y-3 ">
+                    {promptVariables.map(({ key, name, type, options, max_length, required }) => (
+                      <div key={key} className="flex justify-between">
+                        <div className="mr-1 pt-2 shrink-0 w-[120px] text-sm text-gray-900">{name || key}</div>
+                        {type === 'select' && (
+                          <Select
+                            className='w-full'
+                            defaultValue={inputs[key] as string}
+                            onSelect={(i) => { handleInputValueChange(key, i.value as string) }}
+                            items={(options || []).map(i => ({ name: i, value: i }))}
+                            allowSearch={false}
+                            bgClassName='bg-gray-50'
+                          />
+                        )
+                        }
+                        {type === 'string' && (
+                          <input
+                            className="w-full px-3 text-sm leading-9 text-gray-900 border-0 rounded-lg grow h-9 bg-gray-50 focus:outline-none focus:ring-1 focus:ring-inset focus:ring-gray-200"
+                            placeholder={`${name}${!required ? `(${t('appDebug.variableTable.optional')})` : ''}`}
+                            type="text"
+                            value={inputs[key] ? `${inputs[key]}` : ''}
+                            onChange={(e) => { handleInputValueChange(key, e.target.value) }}
+                            maxLength={max_length || DEFAULT_VALUE_MAX_LEN}
+                          />
+                        )}
+                        {type === 'paragraph' && (
+                          <textarea
+                            className="w-full px-3 text-sm leading-9 text-gray-900 border-0 rounded-lg grow h-[120px] bg-gray-50 focus:outline-none focus:ring-1 focus:ring-inset focus:ring-gray-200"
+                            placeholder={`${name}${!required ? `(${t('appDebug.variableTable.optional')})` : ''}`}
+                            value={inputs[key] ? `${inputs[key]}` : ''}
+                            onChange={(e) => { handleInputValueChange(key, e.target.value) }}
+                          />
+                        )}
 
-                        </div>
-                      ))}
-                    </div>
-                  )
-                  : (
-                    <div className='text-xs text-gray-500'>{t('appDebug.inputs.noVar')}</div>
-                  )
-              }
-            </>
-          )
+                      </div>
+                    ))}
+                  </div>
+                )
+                : (
+                  <div className='text-xs text-gray-500'>{t('appDebug.inputs.noVar')}</div>
+                )
+            }
+          </>
+        )
         }
       </div>
 
       {
         appType === AppType.completion && (
-          <div className='px-4'>
-            <div className="mt-3 border-b border-gray-100"></div>
-            <div className="mt-4">
-              <div>
-                <div className="text-[13px] text-gray-900 font-medium">{t('appDebug.inputs.queryTitle')}</div>
-                <div className="mt-2 mb-4 overflow-hidden border border-gray-200 rounded-lg grow bg-gray-50 ">
-                  <div className="px-4 py-2 rounded-t-lg bg-gray-50">
-                    <textarea
-                      rows={4}
-                      className="w-full px-0 text-sm text-gray-900 border-0 bg-gray-50 focus:outline-none placeholder:bg-gray-50"
-                      placeholder={t('appDebug.inputs.queryPlaceholder') as string}
-                      value={value}
-                      onChange={e => onChange && onChange(e.target.value)}
-                    ></textarea>
-                  </div>
-                  <div className="flex items-center justify-between px-3 py-2 bg-gray-50">
-                    <div className="flex pl-0 space-x-1 sm:pl-2">
-                      <span className="bg-gray-100 text-gray-500 text-xs font-medium mr-2 px-2.5 py-0.5 rounded cursor-pointer">{value?.length}</span>
-                    </div>
-                    <Button
-                      type="primary"
-                      onClick={() => onSend && onSend()}
-                      className="w-[80px] !h-8">
-                      <PlayIcon className="shrink-0 w-4 h-4 mr-1" aria-hidden="true" />
-                      <span className='uppercase text-[13px]'>{t('appDebug.inputs.run')}</span>
-                    </Button>
-                  </div>
-                </div>
-              </div>
+          <div>
+            <div className="mt-5 border-b border-gray-100"></div>
+            <div className="flex justify-between mt-4 px-4">
+              <Button
+                className='!h-8 !p-3'
+                onClick={onClear}
+                disabled={false}
+              >
+                <span className='text-[13px]'>{t('common.operation.clear')}</span>
+              </Button>
+
+              {canNotRun
+                ? (<Tooltip
+                  popupContent={t('appDebug.otherError.promptNoBeEmpty')}
+                >
+                  {renderRunButton()}
+                </Tooltip>)
+                : renderRunButton()}
             </div>
           </div>
         )

web/app/components/app/log/list.tsx

@@ -81,7 +81,7 @@ const getFormattedChatList = (messages: ChatMessage[]) => {
   messages.forEach((item: ChatMessage) => {
     newChatList.push({
       id: `question-${item.id}`,
-      content: item.query,
+      content: item.inputs.query || item.inputs.default_input || item.query, // text generation: item.inputs.query; chat: item.query
       isAnswer: false,
     })
 
@@ -288,7 +288,7 @@ const CompletionConversationDetailComp: FC<{ appId?: string; conversationId?: st
       return true
     }
     catch (err) {
-      notify({ type: 'error', message: t('common.actionMsg.modificationFailed') })
+      notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') })
       return false
     }
   }
@@ -301,7 +301,7 @@ const CompletionConversationDetailComp: FC<{ appId?: string; conversationId?: st
       return true
     }
     catch (err) {
-      notify({ type: 'error', message: t('common.actionMsg.modificationFailed') })
+      notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') })
       return false
     }
   }
@@ -332,7 +332,7 @@ const ChatConversationDetailComp: FC<{ appId?: string; conversationId?: string }
       return true
     }
     catch (err) {
-      notify({ type: 'error', message: t('common.actionMsg.modificationFailed') })
+      notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') })
       return false
     }
   }
@@ -344,7 +344,7 @@ const ChatConversationDetailComp: FC<{ appId?: string; conversationId?: string }
       return true
     }
     catch (err) {
-      notify({ type: 'error', message: t('common.actionMsg.modificationFailed') })
+      notify({ type: 'error', message: t('common.actionMsg.modifiedUnsuccessfully') })
       return false
     }
   }
@@ -413,7 +413,7 @@ const ConversationList: FC<IConversationList> = ({ logs, appDetail, onRefresh })
         <tbody className="text-gray-500">
           {logs.data.map((log) => {
             const endUser = log.from_end_user_session_id
-            const leftValue = get(log, isChatMode ? 'summary' : 'message.query')
+            const leftValue = get(log, isChatMode ? 'summary' : 'message.inputs.query') || (!isChatMode ? (get(log, 'message.query') || get(log, 'message.inputs.default_input')) : '') || ''
             const rightValue = get(log, isChatMode ? 'message_count' : 'message.answer')
             return <tr
               key={log.id}

web/app/components/app/overview/appCard.tsx

@@ -119,9 +119,11 @@ function AppCard({
   }
 
   const onGenCode = async () => {
-    setGenLoading(true)
-    await asyncRunSafe(onGenerateCode?.() as any)
-    setGenLoading(false)
+    if (onGenerateCode) {
+      setGenLoading(true)
+      await asyncRunSafe(onGenerateCode())
+      setGenLoading(false)
+    }
   }
 
   return (

web/app/components/app/overview/embedded/index.tsx

@@ -7,6 +7,7 @@ import Modal from '@/app/components/base/modal'
 import copyStyle from '@/app/components/app/chat/copy-btn/style.module.css'
 import Tooltip from '@/app/components/base/tooltip'
 import { useAppContext } from '@/context/app-context'
+import { IS_CE_EDITION } from '@/config'
 
 // const isDevelopment = process.env.NODE_ENV === 'development'
 
@@ -30,7 +31,15 @@ const OPTION_MAP = {
   scripts: {
     getContent: (url: string, token: string, isTestEnv?: boolean) =>
       `<script>
- window.difyChatbotConfig = { token: '${token}'${isTestEnv ? ', isDev: true' : ''} }
+ window.difyChatbotConfig = { 
+  token: '${token}'${isTestEnv
+  ? `, 
+  isDev: true`
+  : ''}${IS_CE_EDITION
+  ? `, 
+  baseUrl: '${url}'`
+  : ''}
+ }
 </script>
 <script
  src="${url}/embed.min.js"

web/app/components/app/overview/settings/index.tsx

@@ -46,7 +46,8 @@ const SettingsModal: FC<ISettingsModalProps> = ({
   onSave,
 }) => {
   const [isShowMore, setIsShowMore] = useState(false)
-  const { title, description, copyright, privacy_policy, default_language, icon, icon_background } = appInfo.site
+  const { icon, icon_background } = appInfo
+  const { title, description, copyright, privacy_policy, default_language } = appInfo.site
   const [inputInfo, setInputInfo] = useState({ title, desc: description, copyright, privacyPolicy: privacy_policy })
   const [language, setLanguage] = useState(default_language)
   const [saveLoading, setSaveLoading] = useState(false)

web/app/components/base/confirm-ui/index.tsx

@@ -41,8 +41,8 @@ const ConfirmUI: FC<IConfirmUIProps> = ({
       </div>
 
       <div className='flex gap-3 mt-4 ml-12'>
-        <div onClick={onConfirm} className='w-20 leading-[34px] text-center text-white border rounded-lg cursor-pointer h-9 border-color-primary-700 bg-primary-700'>{confirmText || t('common.operation.confirm')}</div>
-        <div onClick={onCancel} className='w-20 leading-[34px] text-center text-gray-500 border rounded-lg cursor-pointer h-9 border-color-gray-200'>{cancelText || t('common.operation.cancel')}</div>
+        <div onClick={onConfirm} className='w-20 leading-9 text-center text-white border rounded-lg cursor-pointer h-9 border-color-primary-700 bg-primary-700'>{confirmText || t('common.operation.confirm')}</div>
+        <div onClick={onCancel} className='w-20 leading-9 text-center text-gray-500 border rounded-lg cursor-pointer h-9 border-color-gray-200'>{cancelText || t('common.operation.cancel')}</div>
       </div>
     </div>
 

web/app/components/base/file-icon/index.tsx

@@ -0,0 +1,48 @@
+import type { FC } from 'react'
+import {
+  Csv,
+  Html,
+  Json,
+  Md,
+  Pdf,
+  Txt,
+  Unknow,
+  Xlsx,
+} from '@/app/components/base/icons/src/public/files'
+import { Notion } from '@/app/components/base/icons/src/public/common'
+
+type FileIconProps = {
+  type: string
+  className?: string
+}
+
+const FileIcon: FC<FileIconProps> = ({
+  type,
+  className,
+}) => {
+  switch (type) {
+    case 'csv':
+      return <Csv className={className} />
+    case 'htm':
+    case 'html':
+      return <Html className={className} />
+    case 'json':
+      return <Json className={className} />
+    case 'md':
+    case 'markdown':
+      return <Md className={className} />
+    case 'pdf':
+      return <Pdf className={className} />
+    case 'txt':
+      return <Txt className={className} />
+    case 'xls':
+    case 'xlsx':
+      return <Xlsx className={className} />
+    case 'notion':
+      return <Notion className={className} />
+    default:
+      return <Unknow className={className} />
+  }
+}
+
+export default FileIcon