feat: reset parameters when switch agent strategy

.github/actions/setup-poetry/action.yml

@@ -8,7 +8,7 @@ inputs:
   poetry-version:
     description: Poetry version to set up
     required: true
-    default: '2.0.1'
+    default: '1.8.4'
   poetry-lockfile:
     description: Path to the Poetry lockfile to restore cache from
     required: true

.github/workflows/api-tests.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - plugins/beta
     paths:
       - api/**
       - docker/**
@@ -43,17 +42,25 @@ jobs:
         run: poetry install -C api --with dev
 
       - name: Check dependencies in pyproject.toml
-        run: poetry run -P api bash dev/pytest/pytest_artifacts.sh
+        run: poetry run -C api bash dev/pytest/pytest_artifacts.sh
 
       - name: Run Unit tests
-        run: poetry run -P api bash dev/pytest/pytest_unit_tests.sh
+        run: poetry run -C api bash dev/pytest/pytest_unit_tests.sh
+
+      - name: Run ModelRuntime
+        run: poetry run -C api bash dev/pytest/pytest_model_runtime.sh
 
       - name: Run dify config tests
-        run: poetry run -P api python dev/pytest/pytest_config_tests.py
+        run: poetry run -C api python dev/pytest/pytest_config_tests.py
+
+      - name: Run Tool
+        run: poetry run -C api bash dev/pytest/pytest_tools.sh
 
       - name: Run mypy
         run: |
-          poetry run -C api python -m mypy --install-types --non-interactive .
+          pushd api
+          poetry run python -m mypy --install-types --non-interactive .
+          popd
 
       - name: Set up dotenvs
         run: |
@@ -73,4 +80,4 @@ jobs:
             ssrf_proxy
 
       - name: Run Workflow
-        run: poetry run -P api bash dev/pytest/pytest_workflow.sh
+        run: poetry run -C api bash dev/pytest/pytest_workflow.sh

.github/workflows/build-push.yml

@@ -5,7 +5,6 @@ on:
     branches:
       - "main"
       - "deploy/dev"
-      - "plugins/beta"
       - "dev/plugin-deploy"
   release:
     types: [published]

.github/workflows/db-migration-test.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - plugins/beta
     paths:
       - api/migrations/**
       - .github/workflows/db-migration-test.yml

.github/workflows/style.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
       - main
-      - plugins/beta
 
 concurrency:
   group: style-${{ github.head_ref || github.run_id }}
@@ -39,12 +38,12 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
           poetry run -C api ruff --version
-          poetry run -C api ruff check ./
-          poetry run -C api ruff format --check ./
+          poetry run -C api ruff check ./api
+          poetry run -C api ruff format --check ./api
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: poetry run -P api dotenv-linter ./api/.env.example ./web/.env.example
+        run: poetry run -C api dotenv-linter ./api/.env.example ./web/.env.example
 
       - name: Lint hints
         if: failure()
@@ -67,12 +66,6 @@ jobs:
         with:
           files: web/**
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          version: 10
-          run_install: false
-
       - name: Setup NodeJS
         uses: actions/setup-node@v4
         if: steps.changed-files.outputs.any_changed == 'true'
@@ -87,7 +80,7 @@ jobs:
 
       - name: Web style check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: yarn run lint
+        run: pnpm run lint
 
   docker-compose-template:
     name: Docker Compose Template
@@ -141,7 +134,7 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         env:
           BASH_SEVERITY: warning
-          DEFAULT_BRANCH: plugins/beta
+          DEFAULT_BRANCH: main
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           IGNORE_GENERATED_FILES: true
           IGNORE_GITIGNORED_FILES: true

.github/workflows/vdb-tests.yml

@@ -70,4 +70,4 @@ jobs:
             tidb
 
       - name: Test Vector Stores
-        run: poetry run -P api bash dev/pytest/pytest_vdb.sh
+        run: poetry run -C api bash dev/pytest/pytest_vdb.sh

.gitignore

@@ -197,6 +197,3 @@ api/.vscode
 
 # pnpm
 /.pnpm-store
-
-# plugin migrate
-plugins.jsonl

api/.env.example

@@ -422,8 +422,8 @@ POSITION_PROVIDER_INCLUDES=
 POSITION_PROVIDER_EXCLUDES=
 
 # Plugin configuration
-PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi
-PLUGIN_DAEMON_URL=http://127.0.0.1:5002
+PLUGIN_API_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
+PLUGIN_API_URL=http://127.0.0.1:5002
 PLUGIN_REMOTE_INSTALL_PORT=5003
 PLUGIN_REMOTE_INSTALL_HOST=localhost
 PLUGIN_MAX_PACKAGE_SIZE=15728640
@@ -435,7 +435,7 @@ MARKETPLACE_ENABLED=true
 MARKETPLACE_API_URL=https://marketplace.dify.ai
 
 # Endpoint configuration
-ENDPOINT_URL_TEMPLATE=http://localhost:5002/e/{hook_id}
+ENDPOINT_URL_TEMPLATE=http://localhost/e/{hook_id}
 
 # Reset password token expiry minutes
 RESET_PASSWORD_TOKEN_EXPIRY_MINUTES=5

api/.ruff.toml

@@ -53,12 +53,10 @@ ignore = [
     "FURB152", # math-constant
     "UP007", # non-pep604-annotation
     "UP032", # f-string
-    "UP045", # non-pep604-annotation-optional
     "B005", # strip-with-multi-characters
     "B006", # mutable-argument-default
     "B007", # unused-loop-control-variable
     "B026", # star-arg-unpacking-after-keyword-arg
-    "B903", # class-as-data-structure
     "B904", # raise-without-from-inside-except
     "B905", # zip-without-explicit-strict
     "N806", # non-lowercase-variable-in-function

api/Dockerfile

@@ -4,7 +4,7 @@ FROM python:3.12-slim-bookworm AS base
 WORKDIR /app/api
 
 # Install Poetry
-ENV POETRY_VERSION=2.0.1
+ENV POETRY_VERSION=1.8.4
 
 # if you located in China, you can use aliyun mirror to speed up
 # RUN pip install --no-cache-dir poetry==${POETRY_VERSION} -i https://mirrors.aliyun.com/pypi/simple/
@@ -55,7 +55,6 @@ RUN apt-get update \
     && echo "deb http://deb.debian.org/debian testing main" > /etc/apt/sources.list \
     && apt-get update \
     # For Security
-    # && apt-get install -y --no-install-recommends expat=2.6.4-1 libldap-2.5-0=2.5.19+dfsg-1 perl=5.40.0-8 libsqlite3-0=3.46.1-1 zlib1g=1:1.3.dfsg+really1.3.1-1+b1 \
     # install a chinese font to support the use of tools like matplotlib
     && apt-get install -y fonts-noto-cjk \
     && apt-get autoremove -y \

api/README.md

@@ -79,5 +79,5 @@
 2. Run the tests locally with mocked system environment variables in `tool.pytest_env` section in `pyproject.toml`
 
    ```bash
-   poetry run -P api bash dev/pytest/pytest_all_tests.sh
+   poetry run -C api bash dev/pytest/pytest_all_tests.sh
    ```

api/configs/feature/__init__.py

@@ -141,10 +141,10 @@ class PluginConfig(BaseSettings):
 
     PLUGIN_DAEMON_URL: HttpUrl = Field(
         description="Plugin API URL",
-        default="http://localhost:5002",
+        default="http://plugin:5002",
     )
 
-    PLUGIN_DAEMON_KEY: str = Field(
+    PLUGIN_API_KEY: str = Field(
         description="Plugin API key",
         default="plugin-api-key",
     )
@@ -200,7 +200,7 @@ class EndpointConfig(BaseSettings):
     )
 
     CONSOLE_WEB_URL: str = Field(
-        description="Base URL for the console web interface,used for frontend references and CORS configuration",
+        description="Base URL for the console web interface," "used for frontend references and CORS configuration",
         default="",
     )
 

api/configs/feature/hosted_service/__init__.py

@@ -181,7 +181,7 @@ class HostedFetchAppTemplateConfig(BaseSettings):
     """
 
     HOSTED_FETCH_APP_TEMPLATES_MODE: str = Field(
-        description="Mode for fetching app templates: remote, db, or builtin default to remote,",
+        description="Mode for fetching app templates: remote, db, or builtin" " default to remote,",
         default="remote",
     )
 

api/configs/packaging/__init__.py

@@ -9,7 +9,7 @@ class PackagingInfo(BaseSettings):
 
     CURRENT_VERSION: str = Field(
         description="Dify version",
-        default="1.0.0-beta.1",
+        default="0.15.0",
     )
 
     COMMIT_SHA: str = Field(

api/controllers/console/admin.py

@@ -59,7 +59,7 @@ class InsertExploreAppListApi(Resource):
         with Session(db.engine) as session:
             app = session.execute(select(App).filter(App.id == args["app_id"])).scalar_one_or_none()
         if not app:
-            raise NotFound(f"App '{args['app_id']}' is not found")
+            raise NotFound(f'App \'{args["app_id"]}\' is not found')
 
         site = app.site
         if not site:

api/controllers/console/app/audio.py

@@ -22,7 +22,7 @@ from controllers.console.wraps import account_initialization_required, setup_req
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from core.model_runtime.errors.invoke import InvokeError
 from libs.login import login_required
-from models import App, AppMode
+from models.model import AppMode
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,
@@ -79,7 +79,7 @@ class ChatMessageTextApi(Resource):
     @login_required
     @account_initialization_required
     @get_app_model
-    def post(self, app_model: App):
+    def post(self, app_model):
         from werkzeug.exceptions import InternalServerError
 
         try:
@@ -98,13 +98,9 @@ class ChatMessageTextApi(Resource):
                 and app_model.workflow.features_dict
             ):
                 text_to_speech = app_model.workflow.features_dict.get("text_to_speech")
-                if text_to_speech is None:
-                    raise ValueError("TTS is not enabled")
                 voice = args.get("voice") or text_to_speech.get("voice")
             else:
                 try:
-                    if app_model.app_model_config is None:
-                        raise ValueError("AppModelConfig not found")
                     voice = args.get("voice") or app_model.app_model_config.text_to_speech_dict.get("voice")
                 except Exception:
                     voice = None

api/controllers/console/datasets/data_source.py

@@ -135,7 +135,7 @@ class DataSourceNotionListApi(Resource):
                         data_source_info = json.loads(document.data_source_info)
                         exist_page_ids.append(data_source_info["notion_page_id"])
             # get all authorized pages
-            data_source_bindings = session.scalars(
+            data_source_bindings = session.execute(
                 select(DataSourceOauthBinding).filter_by(
                     tenant_id=current_user.current_tenant_id, provider="notion", disabled=False
                 )

api/controllers/console/datasets/datasets.py

@@ -52,12 +52,12 @@ class DatasetListApi(Resource):
         # provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
-        include_all = request.args.get("include_all", default="false").lower() == "true"
+
         if ids:
             datasets, total = DatasetService.get_datasets_by_ids(ids, current_user.current_tenant_id)
         else:
             datasets, total = DatasetService.get_datasets(
-                page, limit, current_user.current_tenant_id, current_user, search, tag_ids, include_all
+                page, limit, current_user.current_tenant_id, current_user, search, tag_ids
             )
 
         # check embedding setting
@@ -457,7 +457,7 @@ class DatasetIndexingEstimateApi(Resource):
             )
         except LLMBadRequestError:
             raise ProviderNotInitializeError(
-                "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                "No Embedding Model available. Please configure a valid provider " "in the Settings -> Model Provider."
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -619,7 +619,8 @@ class DatasetRetrievalSettingApi(Resource):
         vector_type = dify_config.VECTOR_STORE
         match vector_type:
             case (
-                VectorType.RELYT
+                VectorType.MILVUS
+                | VectorType.RELYT
                 | VectorType.PGVECTOR
                 | VectorType.TIDB_VECTOR
                 | VectorType.CHROMA
@@ -644,7 +645,6 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.TIDB_ON_QDRANT
                 | VectorType.LINDORM
                 | VectorType.COUCHBASE
-                | VectorType.MILVUS
             ):
                 return {
                     "retrieval_method": [

api/controllers/console/datasets/datasets_document.py

@@ -362,7 +362,8 @@ class DatasetInitApi(Resource):
                 )
             except InvokeAuthorizationError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -539,7 +540,8 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
                 return response.model_dump(), 200
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)

api/controllers/console/datasets/datasets_segments.py

@@ -168,7 +168,8 @@ class DatasetDocumentSegmentApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -216,7 +217,8 @@ class DatasetDocumentSegmentAddApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -265,7 +267,8 @@ class DatasetDocumentSegmentUpdateApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -365,9 +368,9 @@ class DatasetDocumentSegmentBatchImportApi(Resource):
             result = []
             for index, row in df.iterrows():
                 if document.doc_form == "qa_model":
-                    data = {"content": row.iloc[0], "answer": row.iloc[1]}
+                    data = {"content": row[0], "answer": row[1]}
                 else:
-                    data = {"content": row.iloc[0]}
+                    data = {"content": row[0]}
                 result.append(data)
             if len(result) == 0:
                 raise ValueError("The CSV file is empty.")
@@ -434,7 +437,8 @@ class ChildChunkAddApi(Resource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)

api/controllers/console/explore/conversation.py

@@ -32,7 +32,7 @@ class ConversationListApi(InstalledAppResource):
 
         pinned = None
         if "pinned" in args and args["pinned"] is not None:
-            pinned = args["pinned"] == "true"
+            pinned = True if args["pinned"] == "true" else False
 
         try:
             with Session(db.engine) as session:

api/controllers/inner_api/wraps.py

@@ -65,7 +65,7 @@ def enterprise_inner_api_user_auth(view):
 def plugin_inner_api_only(view):
     @wraps(view)
     def decorated(*args, **kwargs):
-        if not dify_config.PLUGIN_DAEMON_KEY:
+        if not dify_config.PLUGIN_API_KEY:
             abort(404)
 
         # get header 'X-Inner-Api-Key'

api/controllers/service_api/__init__.py

@@ -7,4 +7,4 @@ api = ExternalApi(bp)
 
 from . import index
 from .app import app, audio, completion, conversation, file, message, workflow
-from .dataset import dataset, document, hit_testing, segment, upload_file
+from .dataset import dataset, document, hit_testing, segment

api/controllers/service_api/dataset/dataset.py

@@ -31,11 +31,8 @@ class DatasetListApi(DatasetApiResource):
         # provider = request.args.get("provider", default="vendor")
         search = request.args.get("keyword", default=None, type=str)
         tag_ids = request.args.getlist("tag_ids")
-        include_all = request.args.get("include_all", default="false").lower() == "true"
 
-        datasets, total = DatasetService.get_datasets(
-            page, limit, tenant_id, current_user, search, tag_ids, include_all
-        )
+        datasets, total = DatasetService.get_datasets(page, limit, tenant_id, current_user, search, tag_ids)
         # check embedding setting
         provider_manager = ProviderManager()
         configurations = provider_manager.get_configurations(tenant_id=current_user.current_tenant_id)

api/controllers/service_api/dataset/segment.py

@@ -53,7 +53,8 @@ class SegmentApi(DatasetApiResource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -94,7 +95,8 @@ class SegmentApi(DatasetApiResource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
@@ -173,7 +175,8 @@ class DatasetSegmentApi(DatasetApiResource):
                 )
             except LLMBadRequestError:
                 raise ProviderNotInitializeError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)

api/controllers/service_api/dataset/upload_file.py

@@ -1,54 +0,0 @@
-from werkzeug.exceptions import NotFound
-
-from controllers.service_api import api
-from controllers.service_api.wraps import (
-    DatasetApiResource,
-)
-from core.file import helpers as file_helpers
-from extensions.ext_database import db
-from models.dataset import Dataset
-from models.model import UploadFile
-from services.dataset_service import DocumentService
-
-
-class UploadFileApi(DatasetApiResource):
-    def get(self, tenant_id, dataset_id, document_id):
-        """Get upload file."""
-        # check dataset
-        dataset_id = str(dataset_id)
-        tenant_id = str(tenant_id)
-        dataset = db.session.query(Dataset).filter(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
-        if not dataset:
-            raise NotFound("Dataset not found.")
-        # check document
-        document_id = str(document_id)
-        document = DocumentService.get_document(dataset.id, document_id)
-        if not document:
-            raise NotFound("Document not found.")
-        # check upload file
-        if document.data_source_type != "upload_file":
-            raise ValueError(f"Document data source type ({document.data_source_type}) is not upload_file.")
-        data_source_info = document.data_source_info_dict
-        if data_source_info and "upload_file_id" in data_source_info:
-            file_id = data_source_info["upload_file_id"]
-            upload_file = db.session.query(UploadFile).filter(UploadFile.id == file_id).first()
-            if not upload_file:
-                raise NotFound("UploadFile not found.")
-        else:
-            raise ValueError("Upload file id not found in document data source info.")
-
-        url = file_helpers.get_signed_file_url(upload_file_id=upload_file.id)
-        return {
-            "id": upload_file.id,
-            "name": upload_file.name,
-            "size": upload_file.size,
-            "extension": upload_file.extension,
-            "url": url,
-            "download_url": f"{url}&as_attachment=true",
-            "mime_type": upload_file.mime_type,
-            "created_by": upload_file.created_by,
-            "created_at": upload_file.created_at.timestamp(),
-        }, 200
-
-
-api.add_resource(UploadFileApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/upload-file")

api/controllers/service_api/wraps.py

@@ -195,11 +195,7 @@ def validate_and_get_api_token(scope: str | None = None):
     with Session(db.engine, expire_on_commit=False) as session:
         update_stmt = (
             update(ApiToken)
-            .where(
-                ApiToken.token == auth_token,
-                (ApiToken.last_used_at.is_(None) | (ApiToken.last_used_at < cutoff_time)),
-                ApiToken.type == scope,
-            )
+            .where(ApiToken.token == auth_token, ApiToken.last_used_at < cutoff_time, ApiToken.type == scope)
             .values(last_used_at=current_time)
             .returning(ApiToken)
         )
@@ -240,7 +236,7 @@ def create_or_update_end_user_for_user_id(app_model: App, user_id: Optional[str]
             tenant_id=app_model.tenant_id,
             app_id=app_model.id,
             type="service_api",
-            is_anonymous=user_id == "DEFAULT-USER",
+            is_anonymous=True if user_id == "DEFAULT-USER" else False,
             session_id=user_id,
         )
         db.session.add(end_user)

api/controllers/web/conversation.py

@@ -39,7 +39,7 @@ class ConversationListApi(WebApiResource):
 
         pinned = None
         if "pinned" in args and args["pinned"] is not None:
-            pinned = args["pinned"] == "true"
+            pinned = True if args["pinned"] == "true" else False
 
         try:
             with Session(db.engine) as session:

api/core/agent/cot_agent_runner.py

@@ -168,7 +168,7 @@ class CotAgentRunner(BaseAgentRunner, ABC):
 
             self.save_agent_thought(
                 agent_thought=agent_thought,
-                tool_name=(scratchpad.action.action_name if scratchpad.action and not scratchpad.is_final() else ""),
+                tool_name=scratchpad.action.action_name if scratchpad.action else "",
                 tool_input={scratchpad.action.action_name: scratchpad.action.action_input} if scratchpad.action else {},
                 tool_invoke_meta={},
                 thought=scratchpad.thought or "",

api/core/app/apps/base_app_queue_manager.py

@@ -167,7 +167,8 @@ class AppQueueManager:
         else:
             if isinstance(data, DeclarativeMeta) or hasattr(data, "_sa_instance_state"):
                 raise TypeError(
-                    "Critical Error: Passing SQLAlchemy Model instances that cause thread safety issues is not allowed."
+                    "Critical Error: Passing SQLAlchemy Model instances "
+                    "that cause thread safety issues is not allowed."
                 )
 
 

api/core/app/apps/message_based_app_generator.py

@@ -89,7 +89,6 @@ class MessageBasedAppGenerator(BaseAppGenerator):
             Conversation.id == conversation_id,
             Conversation.app_id == app_model.id,
             Conversation.status == "normal",
-            Conversation.is_deleted.is_(False),
         ]
 
         if isinstance(user, Account):

api/core/app/apps/workflow_app_runner.py

@@ -241,7 +241,6 @@ class WorkflowBasedAppRunner(AppRunner):
                     predecessor_node_id=event.predecessor_node_id,
                     in_iteration_id=event.in_iteration_id,
                     parallel_mode_run_id=event.parallel_mode_run_id,
-                    agent_strategy=event.agent_strategy,
                 )
             )
         elif isinstance(event, NodeRunSucceededEvent):

api/core/app/entities/queue_entities.py

@@ -6,7 +6,7 @@ from typing import Any, Optional
 from pydantic import BaseModel
 
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk
-from core.workflow.entities.node_entities import AgentNodeStrategyInit, NodeRunMetadataKey
+from core.workflow.entities.node_entities import NodeRunMetadataKey
 from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
 from core.workflow.nodes import NodeType
 from core.workflow.nodes.base import BaseNodeData
@@ -281,7 +281,6 @@ class QueueNodeStartedEvent(AppQueueEvent):
     start_at: datetime
     parallel_mode_run_id: Optional[str] = None
     """iteratoin run in parallel mode run id"""
-    agent_strategy: Optional[AgentNodeStrategyInit] = None
 
 
 class QueueNodeSucceededEvent(AppQueueEvent):

api/core/app/entities/task_entities.py

@@ -6,7 +6,6 @@ from pydantic import BaseModel, ConfigDict
 
 from core.model_runtime.entities.llm_entities import LLMResult
 from core.model_runtime.utils.encoders import jsonable_encoder
-from core.workflow.entities.node_entities import AgentNodeStrategyInit
 from models.workflow import WorkflowNodeExecutionStatus
 
 
@@ -249,7 +248,6 @@ class NodeStartStreamResponse(StreamResponse):
         parent_parallel_start_node_id: Optional[str] = None
         iteration_id: Optional[str] = None
         parallel_run_id: Optional[str] = None
-        agent_strategy: Optional[AgentNodeStrategyInit] = None
 
     event: StreamEvent = StreamEvent.NODE_STARTED
     workflow_run_id: str

api/core/app/task_pipeline/message_cycle_manage.py

@@ -145,7 +145,7 @@ class MessageCycleManage:
 
             # get extension
             if "." in message_file.url:
-                extension = f".{message_file.url.split('.')[-1]}"
+                extension = f'.{message_file.url.split(".")[-1]}'
                 if len(extension) > 10:
                     extension = ".bin"
             else:

api/core/app/task_pipeline/workflow_cycle_manage.py

@@ -541,7 +541,6 @@ class WorkflowCycleManage:
                 parent_parallel_start_node_id=event.parent_parallel_start_node_id,
                 iteration_id=event.in_iteration_id,
                 parallel_run_id=event.parallel_mode_run_id,
-                agent_strategy=event.agent_strategy,
             ),
         )
 

api/core/external_data_tool/api/api.py

@@ -62,9 +62,8 @@ class ApiExternalDataTool(ExternalDataTool):
 
         if not api_based_extension:
             raise ValueError(
-                "[External data tool] API query failed, variable: {}, error: api_based_extension_id is invalid".format(
-                    self.variable
-                )
+                "[External data tool] API query failed, variable: {}, "
+                "error: api_based_extension_id is invalid".format(self.variable)
             )
 
         # decrypt api_key

api/core/file/helpers.py

@@ -33,7 +33,7 @@ def get_signed_file_url_for_plugin(filename: str, mimetype: str, tenant_id: str,
     sign = hmac.new(key, msg.encode(), hashlib.sha256).digest()
     encoded_sign = base64.urlsafe_b64encode(sign).decode()
 
-    return f"{url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}&user_id={user_id}&tenant_id={tenant_id}"
+    return f"{url}?timestamp={timestamp}&nonce={nonce}&sign={encoded_sign}&user_id={user_id}"
 
 
 def verify_plugin_file_signature(

api/core/file/models.py

@@ -90,7 +90,7 @@ class File(BaseModel):
     def markdown(self) -> str:
         url = self.generate_url()
         if self.type == FileType.IMAGE:
-            text = f"![{self.filename or ''}]({url})"
+            text = f'![{self.filename or ""}]({url})'
         else:
             text = f"[{self.filename or url}]({url})"
 

api/core/indexing_runner.py

@@ -530,6 +530,7 @@ class IndexingRunner:
         # chunk nodes by chunk size
         indexing_start_at = time.perf_counter()
         tokens = 0
+        chunk_size = 10
         if dataset_document.doc_form != IndexType.PARENT_CHILD_INDEX:
             # create keyword index
             create_keyword_thread = threading.Thread(
@@ -538,22 +539,11 @@ class IndexingRunner:
             )
             create_keyword_thread.start()
 
-        max_workers = 10
         if dataset.indexing_technique == "high_quality":
-            with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
                 futures = []
-
-                # Distribute documents into multiple groups based on the hash values of page_content
-                # This is done to prevent multiple threads from processing the same document,
-                # Thereby avoiding potential database insertion deadlocks
-                document_groups: list[list[Document]] = [[] for _ in range(max_workers)]
-                for document in documents:
-                    hash = helper.generate_text_hash(document.page_content)
-                    group_index = int(hash, 16) % max_workers
-                    document_groups[group_index].append(document)
-                for chunk_documents in document_groups:
-                    if len(chunk_documents) == 0:
-                        continue
+                for i in range(0, len(documents), chunk_size):
+                    chunk_documents = documents[i : i + chunk_size]
                     futures.append(
                         executor.submit(
                             self._process_chunk,

api/core/llm_generator/prompts.py

@@ -131,7 +131,7 @@ JAVASCRIPT_CODE_GENERATOR_PROMPT_TEMPLATE = (
 SUGGESTED_QUESTIONS_AFTER_ANSWER_INSTRUCTION_PROMPT = (
     "Please help me predict the three most likely questions that human would ask, "
     "and keeping each question under 20 characters.\n"
-    "MAKE SURE your output is the SAME language as the Assistant's latest response. "
+    "MAKE SURE your output is the SAME language as the Assistant's latest response"
     "The output must be an array in JSON format following the specified schema:\n"
     '["question1","question2","question3"]\n'
 )

api/core/model_runtime/model_providers/__base/large_language_model.py

@@ -107,46 +107,11 @@ class LargeLanguageModel(AIModel):
                 content_list = []
                 usage = LLMUsage.empty_usage()
                 system_fingerprint = None
-                tools_calls: list[AssistantPromptMessage.ToolCall] = []
-
-                def increase_tool_call(new_tool_calls: list[AssistantPromptMessage.ToolCall]):
-                    def get_tool_call(tool_name: str):
-                        if not tool_name:
-                            return tools_calls[-1]
-
-                        tool_call = next(
-                            (tool_call for tool_call in tools_calls if tool_call.function.name == tool_name), None
-                        )
-                        if tool_call is None:
-                            tool_call = AssistantPromptMessage.ToolCall(
-                                id="",
-                                type="",
-                                function=AssistantPromptMessage.ToolCall.ToolCallFunction(name=tool_name, arguments=""),
-                            )
-                            tools_calls.append(tool_call)
-
-                        return tool_call
-
-                    for new_tool_call in new_tool_calls:
-                        # get tool call
-                        tool_call = get_tool_call(new_tool_call.function.name)
-                        # update tool call
-                        if new_tool_call.id:
-                            tool_call.id = new_tool_call.id
-                        if new_tool_call.type:
-                            tool_call.type = new_tool_call.type
-                        if new_tool_call.function.name:
-                            tool_call.function.name = new_tool_call.function.name
-                        if new_tool_call.function.arguments:
-                            tool_call.function.arguments += new_tool_call.function.arguments
-
                 for chunk in result:
                     if isinstance(chunk.delta.message.content, str):
                         content += chunk.delta.message.content
                     elif isinstance(chunk.delta.message.content, list):
                         content_list.extend(chunk.delta.message.content)
-                    if chunk.delta.message.tool_calls:
-                        increase_tool_call(chunk.delta.message.tool_calls)
 
                     usage = chunk.delta.usage or LLMUsage.empty_usage()
                     system_fingerprint = chunk.system_fingerprint
@@ -155,10 +120,7 @@ class LargeLanguageModel(AIModel):
                 result = LLMResult(
                     model=model,
                     prompt_messages=prompt_messages,
-                    message=AssistantPromptMessage(
-                        content=content or content_list,
-                        tool_calls=tools_calls,
-                    ),
+                    message=AssistantPromptMessage(content=content or content_list),
                     usage=usage,
                     system_fingerprint=system_fingerprint,
                 )

api/core/model_runtime/model_providers/__base/tokenizers/gpt2_tokenzier.py

@@ -1,9 +1,6 @@
-import logging
 from threading import Lock
 from typing import Any
 
-logger = logging.getLogger(__name__)
-
 _tokenizer: Any = None
 _lock = Lock()
 
@@ -46,6 +43,5 @@ class GPT2Tokenizer:
                     base_path = abspath(__file__)
                     gpt2_tokenizer_path = join(dirname(base_path), "gpt2")
                     _tokenizer = TransformerGPT2Tokenizer.from_pretrained(gpt2_tokenizer_path)
-                    logger.info("Fallback to Transformers' GPT-2 tokenizer from tiktoken")
 
             return _tokenizer

api/core/model_runtime/model_providers/wenxin/llm/ernie-lite-pro-128k.yaml

@@ -0,0 +1,42 @@
+model: ernie-lite-pro-128k
+label:
+  en_US: Ernie-Lite-Pro-128K
+model_type: llm
+features:
+  - agent-thought
+model_properties:
+  mode: chat
+  context_size: 128000
+parameter_rules:
+  - name: temperature
+    use_template: temperature
+    min: 0.1
+    max: 1.0
+    default: 0.8
+  - name: top_p
+    use_template: top_p
+  - name: min_output_tokens
+    label:
+      en_US: "Min Output Tokens"
+      zh_Hans: "最小输出Token数"
+    use_template: max_tokens
+    min: 2
+    max: 2048
+    help:
+      zh_Hans: 指定模型最小输出token数
+      en_US: Specifies the lower limit on the length of generated results.
+  - name: max_output_tokens
+    label:
+      en_US: "Max Output Tokens"
+      zh_Hans: "最大输出Token数"
+    use_template: max_tokens
+    min: 2
+    max: 2048
+    default: 2048
+    help:
+      zh_Hans: 指定模型最大输出token数
+      en_US: Specifies the upper limit on the length of generated results. If the generated results are truncated, you can increase this parameter.
+  - name: presence_penalty
+    use_template: presence_penalty
+  - name: frequency_penalty
+    use_template: frequency_penalty

api/core/model_runtime/schema_validators/common_validator.py

@@ -87,6 +87,6 @@ class CommonValidator:
             if value.lower() not in {"true", "false"}:
                 raise ValueError(f"Variable {credential_form_schema.variable} should be true or false")
 
-            value = value.lower() == "true"
+            value = True if value.lower() == "true" else False
 
         return value

api/core/ops/entities/config_entity.py

@@ -6,7 +6,6 @@ from pydantic import BaseModel, ValidationInfo, field_validator
 class TracingProviderEnum(Enum):
     LANGFUSE = "langfuse"
     LANGSMITH = "langsmith"
-    OPIK = "opik"
 
 
 class BaseTracingConfig(BaseModel):
@@ -57,36 +56,5 @@ class LangSmithConfig(BaseTracingConfig):
         return v
 
 
-class OpikConfig(BaseTracingConfig):
-    """
-    Model class for Opik tracing config.
-    """
-
-    api_key: str | None = None
-    project: str | None = None
-    workspace: str | None = None
-    url: str = "https://www.comet.com/opik/api/"
-
-    @field_validator("project")
-    @classmethod
-    def project_validator(cls, v, info: ValidationInfo):
-        if v is None or v == "":
-            v = "Default Project"
-
-        return v
-
-    @field_validator("url")
-    @classmethod
-    def url_validator(cls, v, info: ValidationInfo):
-        if v is None or v == "":
-            v = "https://www.comet.com/opik/api/"
-        if not v.startswith(("https://", "http://")):
-            raise ValueError("url must start with https:// or http://")
-        if not v.endswith("/api/"):
-            raise ValueError("url should ends with /api/")
-
-        return v
-
-
 OPS_FILE_PATH = "ops_trace/"
 OPS_TRACE_FAILED_KEY = "FAILED_OPS_TRACE"

api/core/ops/opik_trace/opik_trace.py

@@ -1,469 +0,0 @@
-import json
-import logging
-import os
-import uuid
-from datetime import datetime, timedelta
-from typing import Optional, cast
-
-from opik import Opik, Trace
-from opik.id_helpers import uuid4_to_uuid7
-
-from core.ops.base_trace_instance import BaseTraceInstance
-from core.ops.entities.config_entity import OpikConfig
-from core.ops.entities.trace_entity import (
-    BaseTraceInfo,
-    DatasetRetrievalTraceInfo,
-    GenerateNameTraceInfo,
-    MessageTraceInfo,
-    ModerationTraceInfo,
-    SuggestedQuestionTraceInfo,
-    ToolTraceInfo,
-    TraceTaskName,
-    WorkflowTraceInfo,
-)
-from extensions.ext_database import db
-from models.model import EndUser, MessageFile
-from models.workflow import WorkflowNodeExecution
-
-logger = logging.getLogger(__name__)
-
-
-def wrap_dict(key_name, data):
-    """Make sure that the input data is a dict"""
-    if not isinstance(data, dict):
-        return {key_name: data}
-
-    return data
-
-
-def wrap_metadata(metadata, **kwargs):
-    """Add common metatada to all Traces and Spans"""
-    metadata["created_from"] = "dify"
-
-    metadata.update(kwargs)
-
-    return metadata
-
-
-def prepare_opik_uuid(user_datetime: Optional[datetime], user_uuid: Optional[str]):
-    """Opik needs UUIDv7 while Dify uses UUIDv4 for identifier of most
-    messages and objects. The type-hints of BaseTraceInfo indicates that
-    objects start_time and message_id could be null which means we cannot map
-    it to a UUIDv7. Given that we have no way to identify that object
-    uniquely, generate a new random one UUIDv7 in that case.
-    """
-
-    if user_datetime is None:
-        user_datetime = datetime.now()
-
-    if user_uuid is None:
-        user_uuid = str(uuid.uuid4())
-
-    return uuid4_to_uuid7(user_datetime, user_uuid)
-
-
-class OpikDataTrace(BaseTraceInstance):
-    def __init__(
-        self,
-        opik_config: OpikConfig,
-    ):
-        super().__init__(opik_config)
-        self.opik_client = Opik(
-            project_name=opik_config.project,
-            workspace=opik_config.workspace,
-            host=opik_config.url,
-            api_key=opik_config.api_key,
-        )
-        self.project = opik_config.project
-        self.file_base_url = os.getenv("FILES_URL", "http://127.0.0.1:5001")
-
-    def trace(self, trace_info: BaseTraceInfo):
-        if isinstance(trace_info, WorkflowTraceInfo):
-            self.workflow_trace(trace_info)
-        if isinstance(trace_info, MessageTraceInfo):
-            self.message_trace(trace_info)
-        if isinstance(trace_info, ModerationTraceInfo):
-            self.moderation_trace(trace_info)
-        if isinstance(trace_info, SuggestedQuestionTraceInfo):
-            self.suggested_question_trace(trace_info)
-        if isinstance(trace_info, DatasetRetrievalTraceInfo):
-            self.dataset_retrieval_trace(trace_info)
-        if isinstance(trace_info, ToolTraceInfo):
-            self.tool_trace(trace_info)
-        if isinstance(trace_info, GenerateNameTraceInfo):
-            self.generate_name_trace(trace_info)
-
-    def workflow_trace(self, trace_info: WorkflowTraceInfo):
-        dify_trace_id = trace_info.workflow_run_id
-        opik_trace_id = prepare_opik_uuid(trace_info.start_time, dify_trace_id)
-        workflow_metadata = wrap_metadata(
-            trace_info.metadata, message_id=trace_info.message_id, workflow_app_log_id=trace_info.workflow_app_log_id
-        )
-        root_span_id = None
-
-        if trace_info.message_id:
-            dify_trace_id = trace_info.message_id
-            opik_trace_id = prepare_opik_uuid(trace_info.start_time, dify_trace_id)
-
-            trace_data = {
-                "id": opik_trace_id,
-                "name": TraceTaskName.MESSAGE_TRACE.value,
-                "start_time": trace_info.start_time,
-                "end_time": trace_info.end_time,
-                "metadata": workflow_metadata,
-                "input": wrap_dict("input", trace_info.workflow_run_inputs),
-                "output": wrap_dict("output", trace_info.workflow_run_outputs),
-                "tags": ["message", "workflow"],
-                "project_name": self.project,
-            }
-            self.add_trace(trace_data)
-
-            root_span_id = prepare_opik_uuid(trace_info.start_time, trace_info.workflow_run_id)
-            span_data = {
-                "id": root_span_id,
-                "parent_span_id": None,
-                "trace_id": opik_trace_id,
-                "name": TraceTaskName.WORKFLOW_TRACE.value,
-                "input": wrap_dict("input", trace_info.workflow_run_inputs),
-                "output": wrap_dict("output", trace_info.workflow_run_outputs),
-                "start_time": trace_info.start_time,
-                "end_time": trace_info.end_time,
-                "metadata": workflow_metadata,
-                "tags": ["workflow"],
-                "project_name": self.project,
-            }
-            self.add_span(span_data)
-        else:
-            trace_data = {
-                "id": opik_trace_id,
-                "name": TraceTaskName.MESSAGE_TRACE.value,
-                "start_time": trace_info.start_time,
-                "end_time": trace_info.end_time,
-                "metadata": workflow_metadata,
-                "input": wrap_dict("input", trace_info.workflow_run_inputs),
-                "output": wrap_dict("output", trace_info.workflow_run_outputs),
-                "tags": ["workflow"],
-                "project_name": self.project,
-            }
-            self.add_trace(trace_data)
-
-        # through workflow_run_id get all_nodes_execution
-        workflow_nodes_execution_id_records = (
-            db.session.query(WorkflowNodeExecution.id)
-            .filter(WorkflowNodeExecution.workflow_run_id == trace_info.workflow_run_id)
-            .all()
-        )
-
-        for node_execution_id_record in workflow_nodes_execution_id_records:
-            node_execution = (
-                db.session.query(
-                    WorkflowNodeExecution.id,
-                    WorkflowNodeExecution.tenant_id,
-                    WorkflowNodeExecution.app_id,
-                    WorkflowNodeExecution.title,
-                    WorkflowNodeExecution.node_type,
-                    WorkflowNodeExecution.status,
-                    WorkflowNodeExecution.inputs,
-                    WorkflowNodeExecution.outputs,
-                    WorkflowNodeExecution.created_at,
-                    WorkflowNodeExecution.elapsed_time,
-                    WorkflowNodeExecution.process_data,
-                    WorkflowNodeExecution.execution_metadata,
-                )
-                .filter(WorkflowNodeExecution.id == node_execution_id_record.id)
-                .first()
-            )
-
-            if not node_execution:
-                continue
-
-            node_execution_id = node_execution.id
-            tenant_id = node_execution.tenant_id
-            app_id = node_execution.app_id
-            node_name = node_execution.title
-            node_type = node_execution.node_type
-            status = node_execution.status
-            if node_type == "llm":
-                inputs = (
-                    json.loads(node_execution.process_data).get("prompts", {}) if node_execution.process_data else {}
-                )
-            else:
-                inputs = json.loads(node_execution.inputs) if node_execution.inputs else {}
-            outputs = json.loads(node_execution.outputs) if node_execution.outputs else {}
-            created_at = node_execution.created_at or datetime.now()
-            elapsed_time = node_execution.elapsed_time
-            finished_at = created_at + timedelta(seconds=elapsed_time)
-
-            execution_metadata = (
-                json.loads(node_execution.execution_metadata) if node_execution.execution_metadata else {}
-            )
-            metadata = execution_metadata.copy()
-            metadata.update(
-                {
-                    "workflow_run_id": trace_info.workflow_run_id,
-                    "node_execution_id": node_execution_id,
-                    "tenant_id": tenant_id,
-                    "app_id": app_id,
-                    "app_name": node_name,
-                    "node_type": node_type,
-                    "status": status,
-                }
-            )
-
-            process_data = json.loads(node_execution.process_data) if node_execution.process_data else {}
-
-            provider = None
-            model = None
-            total_tokens = 0
-            completion_tokens = 0
-            prompt_tokens = 0
-
-            if process_data and process_data.get("model_mode") == "chat":
-                run_type = "llm"
-                provider = process_data.get("model_provider", None)
-                model = process_data.get("model_name", "")
-                metadata.update(
-                    {
-                        "ls_provider": provider,
-                        "ls_model_name": model,
-                    }
-                )
-
-                try:
-                    if outputs.get("usage"):
-                        total_tokens = outputs["usage"].get("total_tokens", 0)
-                        prompt_tokens = outputs["usage"].get("prompt_tokens", 0)
-                        completion_tokens = outputs["usage"].get("completion_tokens", 0)
-                except Exception:
-                    logger.error("Failed to extract usage", exc_info=True)
-
-            else:
-                run_type = "tool"
-
-            parent_span_id = trace_info.workflow_app_log_id or trace_info.workflow_run_id
-
-            if not total_tokens:
-                total_tokens = execution_metadata.get("total_tokens", 0)
-
-            span_data = {
-                "trace_id": opik_trace_id,
-                "id": prepare_opik_uuid(created_at, node_execution_id),
-                "parent_span_id": prepare_opik_uuid(trace_info.start_time, parent_span_id),
-                "name": node_type,
-                "type": run_type,
-                "start_time": created_at,
-                "end_time": finished_at,
-                "metadata": wrap_metadata(metadata),
-                "input": wrap_dict("input", inputs),
-                "output": wrap_dict("output", outputs),
-                "tags": ["node_execution"],
-                "project_name": self.project,
-                "usage": {
-                    "total_tokens": total_tokens,
-                    "completion_tokens": completion_tokens,
-                    "prompt_tokens": prompt_tokens,
-                },
-                "model": model,
-                "provider": provider,
-            }
-
-            self.add_span(span_data)
-
-    def message_trace(self, trace_info: MessageTraceInfo):
-        # get message file data
-        file_list = cast(list[str], trace_info.file_list) or []
-        message_file_data: Optional[MessageFile] = trace_info.message_file_data
-
-        if message_file_data is not None:
-            file_url = f"{self.file_base_url}/{message_file_data.url}" if message_file_data else ""
-            file_list.append(file_url)
-
-        message_data = trace_info.message_data
-        if message_data is None:
-            return
-
-        metadata = trace_info.metadata
-        message_id = trace_info.message_id
-
-        user_id = message_data.from_account_id
-        metadata["user_id"] = user_id
-        metadata["file_list"] = file_list
-
-        if message_data.from_end_user_id:
-            end_user_data: Optional[EndUser] = (
-                db.session.query(EndUser).filter(EndUser.id == message_data.from_end_user_id).first()
-            )
-            if end_user_data is not None:
-                end_user_id = end_user_data.session_id
-                metadata["end_user_id"] = end_user_id
-
-        trace_data = {
-            "id": prepare_opik_uuid(trace_info.start_time, message_id),
-            "name": TraceTaskName.MESSAGE_TRACE.value,
-            "start_time": trace_info.start_time,
-            "end_time": trace_info.end_time,
-            "metadata": wrap_metadata(metadata),
-            "input": trace_info.inputs,
-            "output": message_data.answer,
-            "tags": ["message", str(trace_info.conversation_mode)],
-            "project_name": self.project,
-        }
-        trace = self.add_trace(trace_data)
-
-        span_data = {
-            "trace_id": trace.id,
-            "name": "llm",
-            "type": "llm",
-            "start_time": trace_info.start_time,
-            "end_time": trace_info.end_time,
-            "metadata": wrap_metadata(metadata),
-            "input": {"input": trace_info.inputs},
-            "output": {"output": message_data.answer},
-            "tags": ["llm", str(trace_info.conversation_mode)],
-            "usage": {
-                "completion_tokens": trace_info.answer_tokens,
-                "prompt_tokens": trace_info.message_tokens,
-                "total_tokens": trace_info.total_tokens,
-            },
-            "project_name": self.project,
-        }
-        self.add_span(span_data)
-
-    def moderation_trace(self, trace_info: ModerationTraceInfo):
-        if trace_info.message_data is None:
-            return
-
-        start_time = trace_info.start_time or trace_info.message_data.created_at
-
-        span_data = {
-            "trace_id": prepare_opik_uuid(start_time, trace_info.message_id),
-            "name": TraceTaskName.MODERATION_TRACE.value,
-            "type": "tool",
-            "start_time": start_time,
-            "end_time": trace_info.end_time or trace_info.message_data.updated_at,
-            "metadata": wrap_metadata(trace_info.metadata),
-            "input": wrap_dict("input", trace_info.inputs),
-            "output": {
-                "action": trace_info.action,
-                "flagged": trace_info.flagged,
-                "preset_response": trace_info.preset_response,
-                "inputs": trace_info.inputs,
-            },
-            "tags": ["moderation"],
-        }
-
-        self.add_span(span_data)
-
-    def suggested_question_trace(self, trace_info: SuggestedQuestionTraceInfo):
-        message_data = trace_info.message_data
-        if message_data is None:
-            return
-
-        start_time = trace_info.start_time or message_data.created_at
-
-        span_data = {
-            "trace_id": prepare_opik_uuid(start_time, trace_info.message_id),
-            "name": TraceTaskName.SUGGESTED_QUESTION_TRACE.value,
-            "type": "tool",
-            "start_time": start_time,
-            "end_time": trace_info.end_time or message_data.updated_at,
-            "metadata": wrap_metadata(trace_info.metadata),
-            "input": wrap_dict("input", trace_info.inputs),
-            "output": wrap_dict("output", trace_info.suggested_question),
-            "tags": ["suggested_question"],
-        }
-
-        self.add_span(span_data)
-
-    def dataset_retrieval_trace(self, trace_info: DatasetRetrievalTraceInfo):
-        if trace_info.message_data is None:
-            return
-
-        start_time = trace_info.start_time or trace_info.message_data.created_at
-
-        span_data = {
-            "trace_id": prepare_opik_uuid(start_time, trace_info.message_id),
-            "name": TraceTaskName.DATASET_RETRIEVAL_TRACE.value,
-            "type": "tool",
-            "start_time": start_time,
-            "end_time": trace_info.end_time or trace_info.message_data.updated_at,
-            "metadata": wrap_metadata(trace_info.metadata),
-            "input": wrap_dict("input", trace_info.inputs),
-            "output": {"documents": trace_info.documents},
-            "tags": ["dataset_retrieval"],
-        }
-
-        self.add_span(span_data)
-
-    def tool_trace(self, trace_info: ToolTraceInfo):
-        span_data = {
-            "trace_id": prepare_opik_uuid(trace_info.start_time, trace_info.message_id),
-            "name": trace_info.tool_name,
-            "type": "tool",
-            "start_time": trace_info.start_time,
-            "end_time": trace_info.end_time,
-            "metadata": wrap_metadata(trace_info.metadata),
-            "input": wrap_dict("input", trace_info.tool_inputs),
-            "output": wrap_dict("output", trace_info.tool_outputs),
-            "tags": ["tool", trace_info.tool_name],
-        }
-
-        self.add_span(span_data)
-
-    def generate_name_trace(self, trace_info: GenerateNameTraceInfo):
-        trace_data = {
-            "id": prepare_opik_uuid(trace_info.start_time, trace_info.message_id),
-            "name": TraceTaskName.GENERATE_NAME_TRACE.value,
-            "start_time": trace_info.start_time,
-            "end_time": trace_info.end_time,
-            "metadata": wrap_metadata(trace_info.metadata),
-            "input": trace_info.inputs,
-            "output": trace_info.outputs,
-            "tags": ["generate_name"],
-            "project_name": self.project,
-        }
-
-        trace = self.add_trace(trace_data)
-
-        span_data = {
-            "trace_id": trace.id,
-            "name": TraceTaskName.GENERATE_NAME_TRACE.value,
-            "start_time": trace_info.start_time,
-            "end_time": trace_info.end_time,
-            "metadata": wrap_metadata(trace_info.metadata),
-            "input": wrap_dict("input", trace_info.inputs),
-            "output": wrap_dict("output", trace_info.outputs),
-            "tags": ["generate_name"],
-        }
-
-        self.add_span(span_data)
-
-    def add_trace(self, opik_trace_data: dict) -> Trace:
-        try:
-            trace = self.opik_client.trace(**opik_trace_data)
-            logger.debug("Opik Trace created successfully")
-            return trace
-        except Exception as e:
-            raise ValueError(f"Opik Failed to create trace: {str(e)}")
-
-    def add_span(self, opik_span_data: dict):
-        try:
-            self.opik_client.span(**opik_span_data)
-            logger.debug("Opik Span created successfully")
-        except Exception as e:
-            raise ValueError(f"Opik Failed to create span: {str(e)}")
-
-    def api_check(self):
-        try:
-            self.opik_client.auth_check()
-            return True
-        except Exception as e:
-            logger.info(f"Opik API check failed: {str(e)}", exc_info=True)
-            raise ValueError(f"Opik API check failed: {str(e)}")
-
-    def get_project_url(self):
-        try:
-            return self.opik_client.get_project_url(project_name=self.project)
-        except Exception as e:
-            logger.info(f"Opik get run url failed: {str(e)}", exc_info=True)
-            raise ValueError(f"Opik get run url failed: {str(e)}")

api/core/ops/ops_trace_manager.py

@@ -17,7 +17,6 @@ from core.ops.entities.config_entity import (
     OPS_FILE_PATH,
     LangfuseConfig,
     LangSmithConfig,
-    OpikConfig,
     TracingProviderEnum,
 )
 from core.ops.entities.trace_entity import (
@@ -33,7 +32,6 @@ from core.ops.entities.trace_entity import (
 )
 from core.ops.langfuse_trace.langfuse_trace import LangFuseDataTrace
 from core.ops.langsmith_trace.langsmith_trace import LangSmithDataTrace
-from core.ops.opik_trace.opik_trace import OpikDataTrace
 from core.ops.utils import get_message_data
 from extensions.ext_database import db
 from extensions.ext_storage import storage
@@ -54,12 +52,6 @@ provider_config_map: dict[str, dict[str, Any]] = {
         "other_keys": ["project", "endpoint"],
         "trace_instance": LangSmithDataTrace,
     },
-    TracingProviderEnum.OPIK.value: {
-        "config_class": OpikConfig,
-        "secret_keys": ["api_key"],
-        "other_keys": ["project", "url", "workspace"],
-        "trace_instance": OpikDataTrace,
-    },
 }
 
 

api/core/plugin/manager/base.py

@@ -30,7 +30,7 @@ from core.plugin.manager.exc import (
 )
 
 plugin_daemon_inner_api_baseurl = dify_config.PLUGIN_DAEMON_URL
-plugin_daemon_inner_api_key = dify_config.PLUGIN_DAEMON_KEY
+plugin_daemon_inner_api_key = dify_config.PLUGIN_API_KEY
 
 T = TypeVar("T", bound=(BaseModel | dict | list | bool | str))
 

api/core/plugin/manager/tool.py

@@ -48,10 +48,8 @@ class PluginToolManager(BasePluginManager):
         tool_provider_id = GenericProviderID(provider)
 
         def transformer(json_response: dict[str, Any]) -> dict:
-            data = json_response.get("data")
-            if data:
-                for tool in data.get("declaration", {}).get("tools", []):
-                    tool["identity"]["provider"] = tool_provider_id.provider_name
+            for tool in json_response.get("data", {}).get("declaration", {}).get("tools", []):
+                tool["identity"]["provider"] = tool_provider_id.provider_name
 
             return json_response
 

api/core/provider_manager.py

@@ -23,12 +23,7 @@ from core.helper import encrypter
 from core.helper.model_provider_cache import ProviderCredentialsCache, ProviderCredentialsCacheType
 from core.helper.position_helper import is_filtered
 from core.model_runtime.entities.model_entities import ModelType
-from core.model_runtime.entities.provider_entities import (
-    ConfigurateMethod,
-    CredentialFormSchema,
-    FormType,
-    ProviderEntity,
-)
+from core.model_runtime.entities.provider_entities import CredentialFormSchema, FormType, ProviderEntity
 from core.model_runtime.model_providers.model_provider_factory import ModelProviderFactory
 from extensions import ext_hosting_provider
 from extensions.ext_database import db
@@ -844,18 +839,11 @@ class ProviderManager:
         :return:
         """
         # Get provider model credential secret variables
-        if ConfigurateMethod.PREDEFINED_MODEL in provider_entity.configurate_methods:
-            model_credential_secret_variables = self._extract_secret_variables(
-                provider_entity.provider_credential_schema.credential_form_schemas
-                if provider_entity.provider_credential_schema
-                else []
-            )
-        else:
-            model_credential_secret_variables = self._extract_secret_variables(
-                provider_entity.model_credential_schema.credential_form_schemas
-                if provider_entity.model_credential_schema
-                else []
-            )
+        model_credential_secret_variables = self._extract_secret_variables(
+            provider_entity.model_credential_schema.credential_form_schemas
+            if provider_entity.model_credential_schema
+            else []
+        )
 
         model_settings: list[ModelSettings] = []
         if not provider_model_settings:

api/core/rag/datasource/vdb/lindorm/lindorm_vector.py

@@ -258,7 +258,7 @@ class LindormVectorStore(BaseVector):
             hnsw_ef_construction = kwargs.pop("hnsw_ef_construction", 500)
             ivfpq_m = kwargs.pop("ivfpq_m", dimension)
             nlist = kwargs.pop("nlist", 1000)
-            centroids_use_hnsw = kwargs.pop("centroids_use_hnsw", nlist >= 5000)
+            centroids_use_hnsw = kwargs.pop("centroids_use_hnsw", True if nlist >= 5000 else False)
             centroids_hnsw_m = kwargs.pop("centroids_hnsw_m", 24)
             centroids_hnsw_ef_construct = kwargs.pop("centroids_hnsw_ef_construct", 500)
             centroids_hnsw_ef_search = kwargs.pop("centroids_hnsw_ef_search", 100)
@@ -305,7 +305,7 @@ def default_text_mapping(dimension: int, method_name: str, **kwargs: Any) -> dic
     if method_name == "ivfpq":
         ivfpq_m = kwargs["ivfpq_m"]
         nlist = kwargs["nlist"]
-        centroids_use_hnsw = nlist > 10000
+        centroids_use_hnsw = True if nlist > 10000 else False
         centroids_hnsw_m = 24
         centroids_hnsw_ef_construct = 500
         centroids_hnsw_ef_search = 100

api/core/rag/datasource/vdb/pgvector/pgvector.py

@@ -57,11 +57,6 @@ CREATE TABLE IF NOT EXISTS {table_name} (
 ) using heap;
 """
 
-SQL_CREATE_INDEX = """
-CREATE INDEX IF NOT EXISTS embedding_cosine_v1_idx ON {table_name} 
-USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64);
-"""
-
 
 class PGVector(BaseVector):
     def __init__(self, collection_name: str, config: PGVectorConfig):
@@ -210,10 +205,7 @@ class PGVector(BaseVector):
             with self._get_cursor() as cur:
                 cur.execute("CREATE EXTENSION IF NOT EXISTS vector")
                 cur.execute(SQL_CREATE_TABLE.format(table_name=self.table_name, dimension=dimension))
-                # PG hnsw index only support 2000 dimension or less
-                # ref: https://github.com/pgvector/pgvector?tab=readme-ov-file#indexing
-                if dimension <= 2000:
-                    cur.execute(SQL_CREATE_INDEX.format(table_name=self.table_name))
+                # TODO: create index https://github.com/pgvector/pgvector?tab=readme-ov-file#indexing
             redis_client.set(collection_exist_cache_key, 1, ex=3600)
 
 

api/core/rag/embedding/cached_embedding.py

@@ -74,7 +74,7 @@ class CacheEmbedding(Embeddings):
                             embedding_queue_embeddings.append(normalized_embedding)
                         except IntegrityError:
                             db.session.rollback()
-                        except Exception:
+                        except Exception as e:
                             logging.exception("Failed transform embedding")
                 cache_embeddings = []
                 try:

api/core/rag/extractor/firecrawl/firecrawl_app.py

@@ -31,7 +31,7 @@ class FirecrawlApp:
                     "markdown": data.get("markdown"),
                 }
             else:
-                raise Exception(f"Failed to scrape URL. Error: {response_data['error']}")
+                raise Exception(f'Failed to scrape URL. Error: {response_data["error"]}')
 
         elif response.status_code in {402, 409, 500}:
             error_message = response.json().get("error", "Unknown error occurred")

api/core/rag/extractor/notion_extractor.py

@@ -358,7 +358,8 @@ class NotionExtractor(BaseExtractor):
 
         if not data_source_binding:
             raise Exception(
-                f"No notion data source binding found for tenant {tenant_id} and notion workspace {notion_workspace_id}"
+                f"No notion data source binding found for tenant {tenant_id} "
+                f"and notion workspace {notion_workspace_id}"
             )
 
         return cast(str, data_source_binding.access_token)

api/core/rag/extractor/word_extractor.py

@@ -50,7 +50,7 @@ class WordExtractor(BaseExtractor):
 
             self.web_path = self.file_path
             # TODO: use a better way to handle the file
-            self.temp_file = tempfile.NamedTemporaryFile()  # noqa SIM115
+            self.temp_file = tempfile.NamedTemporaryFile()  # noqa: SIM115
             self.temp_file.write(r.content)
             self.file_path = self.temp_file.name
         elif not os.path.isfile(self.file_path):

api/core/rag/index_processor/processor/qa_index_processor.py

@@ -112,7 +112,7 @@ class QAIndexProcessor(BaseIndexProcessor):
             df = pd.read_csv(file)
             text_docs = []
             for index, row in df.iterrows():
-                data = Document(page_content=row.iloc[0], metadata={"answer": row.iloc[1]})
+                data = Document(page_content=row[0], metadata={"answer": row[1]})
                 text_docs.append(data)
             if len(text_docs) == 0:
                 raise ValueError("The CSV file is empty.")

api/core/tools/builtin_tool/providers/time/tools/timezone_conversion.py

@@ -48,6 +48,6 @@ class TimezoneConversionTool(BuiltinTool):
             datetime_with_tz = input_timezone.localize(local_time)
             # timezone convert
             converted_datetime = datetime_with_tz.astimezone(output_timezone)
-            return converted_datetime.strftime(format=time_format)  # type: ignore
+            return converted_datetime.strftime(format=time_format)
         except Exception as e:
             raise ToolInvokeError(str(e))

api/core/tools/builtin_tool/providers/webscraper/webscraper.py

@@ -5,7 +5,4 @@ from core.tools.builtin_tool.provider import BuiltinToolProviderController
 
 class WebscraperProvider(BuiltinToolProviderController):
     def _validate_credentials(self, user_id: str, credentials: dict[str, Any]) -> None:
-        """
-        Validate credentials
-        """
         pass

api/core/tools/custom_tool/tool.py

@@ -94,9 +94,9 @@ class ApiTool(Tool):
             if "api_key_header_prefix" in credentials:
                 api_key_header_prefix = credentials["api_key_header_prefix"]
                 if api_key_header_prefix == "basic" and credentials["api_key_value"]:
-                    credentials["api_key_value"] = f"Basic {credentials['api_key_value']}"
+                    credentials["api_key_value"] = f'Basic {credentials["api_key_value"]}'
                 elif api_key_header_prefix == "bearer" and credentials["api_key_value"]:
-                    credentials["api_key_value"] = f"Bearer {credentials['api_key_value']}"
+                    credentials["api_key_value"] = f'Bearer {credentials["api_key_value"]}'
                 elif api_key_header_prefix == "custom":
                     pass
 

api/core/tools/plugin_tool/provider.py

@@ -48,9 +48,7 @@ class PluginToolProviderController(BuiltinToolProviderController):
         """
         return tool with given name
         """
-        tool_entity = next(
-            (tool_entity for tool_entity in self.entity.tools if tool_entity.identity.name == tool_name), None
-        )
+        tool_entity = next(tool_entity for tool_entity in self.entity.tools if tool_entity.identity.name == tool_name)
 
         if not tool_entity:
             raise ValueError(f"Tool with name {tool_name} not found")

api/core/tools/utils/message_transformer.py

@@ -39,7 +39,7 @@ class ToolFileMessageTransformer:
                         conversation_id=conversation_id,
                     )
 
-                    url = f"/files/tools/{file.id}{guess_extension(file.mimetype) or '.png'}"
+                    url = f'/files/tools/{file.id}{guess_extension(file.mimetype) or ".png"}'
 
                     yield ToolInvokeMessage(
                         type=ToolInvokeMessage.MessageType.IMAGE_LINK,
@@ -115,4 +115,4 @@ class ToolFileMessageTransformer:
 
     @classmethod
     def get_tool_file_url(cls, tool_file_id: str, extension: Optional[str]) -> str:
-        return f"/files/tools/{tool_file_id}{extension or '.bin'}"
+        return f'/files/tools/{tool_file_id}{extension or ".bin"}'

api/core/tools/utils/parser.py

@@ -5,7 +5,6 @@ from json import loads as json_loads
 from json.decoder import JSONDecodeError
 from typing import Optional
 
-from flask import request
 from requests import get
 from yaml import YAMLError, safe_load  # type: ignore
 
@@ -30,10 +29,6 @@ class ApiBasedToolSchemaParser:
             raise ToolProviderNotFoundError("No server found in the openapi yaml.")
 
         server_url = openapi["servers"][0]["url"]
-        request_env = request.headers.get("X-Request-Env")
-        if request_env:
-            matched_servers = [server["url"] for server in openapi["servers"] if server["env"] == request_env]
-            server_url = matched_servers[0] if matched_servers else server_url
 
         # list all interfaces
         interfaces = []
@@ -117,7 +112,7 @@ class ApiBasedToolSchemaParser:
                                 llm_description=property.get("description", ""),
                                 default=property.get("default", None),
                                 placeholder=I18nObject(
-                                    en_US=property.get("description", ""), zh_Hans=property.get("description", "")
+                                    en_US=parameter.get("description", ""), zh_Hans=parameter.get("description", "")
                                 ),
                             )
 
@@ -149,7 +144,7 @@ class ApiBasedToolSchemaParser:
                 if not path:
                     path = str(uuid.uuid4())
 
-                interface["operation"]["operationId"] = f"{path}_{interface['method']}"
+                interface["operation"]["operationId"] = f'{path}_{interface["method"]}'
 
             bundles.append(
                 ApiToolBundle(

api/core/variables/segments.py

@@ -134,10 +134,6 @@ class ArrayStringSegment(ArraySegment):
     value_type: SegmentType = SegmentType.ARRAY_STRING
     value: Sequence[str]
 
-    @property
-    def text(self) -> str:
-        return json.dumps(self.value)
-
 
 class ArrayNumberSegment(ArraySegment):
     value_type: SegmentType = SegmentType.ARRAY_NUMBER

api/core/workflow/entities/node_entities.py

@@ -49,8 +49,3 @@ class NodeRunResult(BaseModel):
 
     # single step node run retry
     retry_index: int = 0
-
-
-class AgentNodeStrategyInit(BaseModel):
-    name: str
-    icon: str | None = None

api/core/workflow/graph_engine/entities/event.py

@@ -4,7 +4,6 @@ from typing import Any, Optional
 
 from pydantic import BaseModel, Field
 
-from core.workflow.entities.node_entities import AgentNodeStrategyInit
 from core.workflow.graph_engine.entities.runtime_route_state import RouteNodeState
 from core.workflow.nodes import NodeType
 from core.workflow.nodes.base import BaseNodeData
@@ -67,10 +66,8 @@ class BaseNodeEvent(GraphEngineEvent):
 
 class NodeRunStartedEvent(BaseNodeEvent):
     predecessor_node_id: Optional[str] = None
-    """predecessor node id"""
     parallel_mode_run_id: Optional[str] = None
-    """iteration node parallel mode run id"""
-    agent_strategy: Optional[AgentNodeStrategyInit] = None
+    """predecessor node id"""
 
 
 class NodeRunStreamChunkEvent(BaseNodeEvent):

api/core/workflow/graph_engine/graph_engine.py

@@ -14,7 +14,7 @@ from flask import Flask, current_app
 from configs import dify_config
 from core.app.apps.base_app_queue_manager import GenerateTaskStoppedError
 from core.app.entities.app_invoke_entities import InvokeFrom
-from core.workflow.entities.node_entities import AgentNodeStrategyInit, NodeRunMetadataKey, NodeRunResult
+from core.workflow.entities.node_entities import NodeRunMetadataKey, NodeRunResult
 from core.workflow.entities.variable_pool import VariablePool, VariableValue
 from core.workflow.graph_engine.condition_handlers.condition_manager import ConditionManager
 from core.workflow.graph_engine.entities.event import (
@@ -40,8 +40,6 @@ from core.workflow.graph_engine.entities.graph_init_params import GraphInitParam
 from core.workflow.graph_engine.entities.graph_runtime_state import GraphRuntimeState
 from core.workflow.graph_engine.entities.runtime_route_state import RouteNodeState
 from core.workflow.nodes import NodeType
-from core.workflow.nodes.agent.agent_node import AgentNode
-from core.workflow.nodes.agent.entities import AgentNodeData
 from core.workflow.nodes.answer.answer_stream_processor import AnswerStreamProcessor
 from core.workflow.nodes.answer.base_stream_processor import StreamProcessor
 from core.workflow.nodes.base import BaseNode
@@ -608,14 +606,6 @@ class GraphEngine:
         Run node
         """
         # trigger node run start event
-        agent_strategy = (
-            AgentNodeStrategyInit(
-                name=cast(AgentNodeData, node_instance.node_data).agent_strategy_name,
-                icon=cast(AgentNode, node_instance).agent_strategy_icon,
-            )
-            if node_instance.node_type == NodeType.AGENT
-            else None
-        )
         yield NodeRunStartedEvent(
             id=node_instance.id,
             node_id=node_instance.node_id,
@@ -627,7 +617,6 @@ class GraphEngine:
             parallel_start_node_id=parallel_start_node_id,
             parent_parallel_id=parent_parallel_id,
             parent_parallel_start_node_id=parent_parallel_start_node_id,
-            agent_strategy=agent_strategy,
         )
 
         db.session.close()

api/core/workflow/nodes/agent/agent_node.py

@@ -8,12 +8,12 @@ from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
 from core.plugin.manager.exc import PluginDaemonClientSideError
 from core.plugin.manager.plugin import PluginInstallationManager
-from core.tools.entities.tool_entities import ToolParameter, ToolProviderType
+from core.tools.entities.tool_entities import ToolProviderType
 from core.tools.tool_manager import ToolManager
 from core.workflow.entities.node_entities import NodeRunResult
 from core.workflow.entities.variable_pool import VariablePool
 from core.workflow.enums import SystemVariableKey
-from core.workflow.nodes.agent.entities import AgentNodeData, ParamsAutoGenerated
+from core.workflow.nodes.agent.entities import AgentNodeData
 from core.workflow.nodes.base.entities import BaseNodeData
 from core.workflow.nodes.enums import NodeType
 from core.workflow.nodes.event.event import RunCompletedEvent
@@ -90,11 +90,18 @@ class AgentNode(ToolNode):
 
         try:
             # convert tool messages
-
+            manager = PluginInstallationManager()
+            plugins = manager.list_plugins(self.tenant_id)
+            current_plugin = next(
+                plugin
+                for plugin in plugins
+                if f"{plugin.plugin_id}/{plugin.name}"
+                == cast(AgentNodeData, self.node_data).agent_strategy_provider_name
+            )
             yield from self._transform_message(
                 message_stream,
                 {
-                    "icon": self.agent_strategy_icon,
+                    "icon": current_plugin.declaration.icon,
                     "agent_strategy": cast(AgentNodeData, self.node_data).agent_strategy_name,
                 },
                 parameters_for_log,
@@ -161,29 +168,11 @@ class AgentNode(ToolNode):
                     value = cast(list[dict[str, Any]], value)
                     tool_value = []
                     for tool in value:
-                        provider_type = ToolProviderType(tool.get("type", ToolProviderType.BUILT_IN.value))
-                        # handle the original settings
-                        original_parameters = tool.get("parameters", {})
-                        setting_params = tool.get("settings", {})
-                        manual_input_params = []
-                        # handle legacy data compatibility
-                        if not all(isinstance(v, dict) for _, v in original_parameters.items()):
-                            parameters = original_parameters
-                        else:
-                            params = {}
-                            for key, param in original_parameters.items():
-                                if param.get("auto", ParamsAutoGenerated.OPEN.value) == ParamsAutoGenerated.CLOSE.value:
-                                    params[key] = param.get("value", "")
-                                    manual_input_params.append(key)
-                                else:
-                                    params[key] = None
-                            settings = {k: v.get("value", None) for k, v in setting_params.items()}
-                            parameters = {**params, **settings}
                         entity = AgentToolEntity(
                             provider_id=tool.get("provider_name", ""),
-                            provider_type=provider_type,
+                            provider_type=ToolProviderType.BUILT_IN,
                             tool_name=tool.get("tool_name", ""),
-                            tool_parameters=parameters,
+                            tool_parameters=tool.get("parameters", {}),
                             plugin_unique_identifier=tool.get("plugin_unique_identifier", None),
                         )
 
@@ -196,27 +185,13 @@ class AgentNode(ToolNode):
                             tool_runtime.entity.description.llm = (
                                 extra.get("descrption", "") or tool_runtime.entity.description.llm
                             )
-                        for params in tool_runtime.entity.parameters:
-                            params.form = (
-                                ToolParameter.ToolParameterForm.FORM
-                                if params.name in manual_input_params
-                                else params.form
-                            )
-                        if tool_runtime.entity.parameters:
-                            manual_input_value = {
-                                key: value for key, value in parameters.items() if key in manual_input_params
+
+                        tool_value.append(
+                            {
+                                **tool_runtime.entity.model_dump(mode="json"),
+                                "runtime_parameters": tool_runtime.runtime.runtime_parameters,
                             }
-                            runtime_parameters = {
-                                **tool_runtime.runtime.runtime_parameters,
-                                **manual_input_value,
-                            }
-                            tool_value.append(
-                                {
-                                    **tool_runtime.entity.model_dump(mode="json"),
-                                    "runtime_parameters": runtime_parameters,
-                                    "provider_type": provider_type.value,
-                                }
-                            )
+                        )
                     value = tool_value
                 if parameter.type == "model-selector":
                     value = cast(dict[str, Any], value)
@@ -254,33 +229,16 @@ class AgentNode(ToolNode):
         result: dict[str, Any] = {}
         for parameter_name in node_data.agent_parameters:
             input = node_data.agent_parameters[parameter_name]
-            if input.type in ["mixed", "constant"]:
-                selectors = VariableTemplateParser(str(input.value)).extract_variable_selectors()
+            if input.type == "mixed":
+                assert isinstance(input.value, str)
+                selectors = VariableTemplateParser(input.value).extract_variable_selectors()
                 for selector in selectors:
                     result[selector.variable] = selector.value_selector
             elif input.type == "variable":
                 result[parameter_name] = input.value
+            elif input.type == "constant":
+                pass
 
         result = {node_id + "." + key: value for key, value in result.items()}
 
         return result
-
-    @property
-    def agent_strategy_icon(self) -> str | None:
-        """
-        Get agent strategy icon
-        :return:
-        """
-        manager = PluginInstallationManager()
-        plugins = manager.list_plugins(self.tenant_id)
-        try:
-            current_plugin = next(
-                plugin
-                for plugin in plugins
-                if f"{plugin.plugin_id}/{plugin.name}"
-                == cast(AgentNodeData, self.node_data).agent_strategy_provider_name
-            )
-            icon = current_plugin.declaration.icon
-        except StopIteration:
-            icon = None
-        return icon

api/core/workflow/nodes/agent/entities.py

@@ -1,4 +1,3 @@
-from enum import Enum
 from typing import Any, Literal, Union
 
 from pydantic import BaseModel
@@ -17,8 +16,3 @@ class AgentNodeData(BaseNodeData):
         type: Literal["mixed", "variable", "constant"]
 
     agent_parameters: dict[str, AgentInput]
-
-
-class ParamsAutoGenerated(Enum):
-    CLOSE = 0
-    OPEN = 1

api/core/workflow/nodes/answer/base_stream_processor.py

@@ -1,7 +1,6 @@
 import logging
 from abc import ABC, abstractmethod
 from collections.abc import Generator
-from typing import Optional
 
 from core.workflow.entities.variable_pool import VariablePool
 from core.workflow.graph_engine.entities.event import GraphEngineEvent, NodeRunExceptionEvent, NodeRunSucceededEvent
@@ -49,35 +48,25 @@ class StreamProcessor(ABC):
                     # we remove the node maybe shortcut the answer node, so comment this code for now
                     # there is not effect on the answer node and the workflow, when we have a better solution
                     # we can open this code. Issues: #11542 #9560 #10638 #10564
-                    # ids = self._fetch_node_ids_in_reachable_branch(edge.target_node_id)
-                    # if "answer" in ids:
-                    #     continue
-                    # else:
-                    #     reachable_node_ids.extend(ids)
-
-                    # The branch_identify parameter is added to ensure that
-                    # only nodes in the correct logical branch are included.
-                    ids = self._fetch_node_ids_in_reachable_branch(edge.target_node_id, run_result.edge_source_handle)
-                    reachable_node_ids.extend(ids)
+                    ids = self._fetch_node_ids_in_reachable_branch(edge.target_node_id)
+                    if "answer" in ids:
+                        continue
+                    else:
+                        reachable_node_ids.extend(ids)
                 else:
                     unreachable_first_node_ids.append(edge.target_node_id)
 
             for node_id in unreachable_first_node_ids:
                 self._remove_node_ids_in_unreachable_branch(node_id, reachable_node_ids)
 
-    def _fetch_node_ids_in_reachable_branch(self, node_id: str, branch_identify: Optional[str] = None) -> list[str]:
+    def _fetch_node_ids_in_reachable_branch(self, node_id: str) -> list[str]:
         node_ids = []
         for edge in self.graph.edge_mapping.get(node_id, []):
             if edge.target_node_id == self.graph.root_node_id:
                 continue
 
-            # Only follow edges that match the branch_identify or have no run_condition
-            if edge.run_condition and edge.run_condition.branch_identify:
-                if not branch_identify or edge.run_condition.branch_identify != branch_identify:
-                    continue
-
             node_ids.append(edge.target_node_id)
-            node_ids.extend(self._fetch_node_ids_in_reachable_branch(edge.target_node_id, branch_identify))
+            node_ids.extend(self._fetch_node_ids_in_reachable_branch(edge.target_node_id))
         return node_ids
 
     def _remove_node_ids_in_unreachable_branch(self, node_id: str, reachable_node_ids: list[str]) -> None:

api/core/workflow/nodes/end/end_stream_generate_router.py

@@ -64,7 +64,7 @@ class EndStreamGeneratorRouter:
                 node_type = node.get("data", {}).get("type")
                 if (
                     variable_selector.value_selector not in value_selectors
-                    and (node_type in (NodeType.LLM.value, NodeType.AGENT.value))
+                    and node_type == NodeType.LLM.value
                     and variable_selector.value_selector[1] == "text"
                 ):
                     value_selectors.append(list(variable_selector.value_selector))

api/core/workflow/nodes/http_request/executor.py

@@ -253,9 +253,9 @@ class Executor:
         )
         if executor_response.size > threshold_size:
             raise ResponseSizeError(
-                f"{'File' if executor_response.is_file else 'Text'} size is too large,"
-                f" max size is {threshold_size / 1024 / 1024:.2f} MB,"
-                f" but current size is {executor_response.readable_size}."
+                f'{"File" if executor_response.is_file else "Text"} size is too large,'
+                f' max size is {threshold_size / 1024 / 1024:.2f} MB,'
+                f' but current size is {executor_response.readable_size}.'
             )
 
         return executor_response
@@ -338,7 +338,7 @@ class Executor:
                 if self.auth.config and self.auth.config.header:
                     authorization_header = self.auth.config.header
                 if k.lower() == authorization_header.lower():
-                    raw += f"{k}: {'*' * len(v)}\r\n"
+                    raw += f'{k}: {"*" * len(v)}\r\n'
                     continue
             raw += f"{k}: {v}\r\n"
 

api/core/workflow/nodes/variable_assigner/v2/node.py

@@ -1,5 +1,4 @@
 import json
-from collections.abc import Sequence
 from typing import Any, cast
 
 from core.variables import SegmentType, Variable
@@ -32,7 +31,7 @@ class VariableAssignerNode(BaseNode[VariableAssignerNodeData]):
         inputs = self.node_data.model_dump()
         process_data: dict[str, Any] = {}
         # NOTE: This node has no outputs
-        updated_variable_selectors: list[Sequence[str]] = []
+        updated_variables: list[Variable] = []
 
         try:
             for item in self.node_data.items:
@@ -99,8 +98,7 @@ class VariableAssignerNode(BaseNode[VariableAssignerNodeData]):
                     value=item.value,
                 )
                 variable = variable.model_copy(update={"value": updated_value})
-                self.graph_runtime_state.variable_pool.add(variable.selector, variable)
-                updated_variable_selectors.append(variable.selector)
+                updated_variables.append(variable)
         except VariableOperatorNodeError as e:
             return NodeRunResult(
                 status=WorkflowNodeExecutionStatus.FAILED,
@@ -109,15 +107,9 @@ class VariableAssignerNode(BaseNode[VariableAssignerNodeData]):
                 error=str(e),
             )
 
-        # The `updated_variable_selectors` is a list contains list[str] which not hashable,
-        # remove the duplicated items first.
-        updated_variable_selectors = list(set(map(tuple, updated_variable_selectors)))
-
         # Update variables
-        for selector in updated_variable_selectors:
-            variable = self.graph_runtime_state.variable_pool.get(selector)
-            if not isinstance(variable, Variable):
-                raise VariableNotFoundError(variable_selector=selector)
+        for variable in updated_variables:
+            self.graph_runtime_state.variable_pool.add(variable.selector, variable)
             process_data[variable.name] = variable.value
 
             if variable.selector[0] == CONVERSATION_VARIABLE_NODE_ID:

api/events/event_handlers/delete_tool_parameters_cache_when_sync_draft_workflow.py

@@ -26,7 +26,7 @@ def handle(sender, **kwargs):
                     tool_runtime=tool_runtime,
                     provider_name=tool_entity.provider_name,
                     provider_type=tool_entity.provider_type,
-                    identity_id=f"WORKFLOW.{app.id}.{node_data.get('id')}",
+                    identity_id=f'WORKFLOW.{app.id}.{node_data.get("id")}',
                 )
                 manager.delete_tool_parameters_cache()
             except:

api/extensions/storage/opendal_storage.py

@@ -34,7 +34,7 @@ class OpenDALStorage(BaseStorage):
             root = kwargs.get("root", "storage")
             Path(root).mkdir(parents=True, exist_ok=True)
 
-        self.op = opendal.Operator(scheme=scheme, **kwargs)  # type: ignore
+        self.op = opendal.Operator(scheme=scheme, **kwargs)
         logger.debug(f"opendal operator created with scheme {scheme}")
         retry_layer = opendal.layers.RetryLayer(max_times=3, factor=2.0, jitter=True)
         self.op = self.op.layer(retry_layer)

api/fields/hit_testing_fields.py

@@ -14,7 +14,6 @@ segment_fields = {
     "position": fields.Integer,
     "document_id": fields.String,
     "content": fields.String,
-    "sign_content": fields.String,
     "answer": fields.String,
     "word_count": fields.Integer,
     "tokens": fields.Integer,

api/fields/member_fields.py

@@ -1,6 +1,6 @@
 from flask_restful import fields  # type: ignore
 
-from libs.helper import AvatarUrlField, TimestampField
+from libs.helper import TimestampField
 
 simple_account_fields = {"id": fields.String, "name": fields.String, "email": fields.String}
 
@@ -8,7 +8,6 @@ account_fields = {
     "id": fields.String,
     "name": fields.String,
     "avatar": fields.String,
-    "avatar_url": AvatarUrlField,
     "email": fields.String,
     "is_password_set": fields.Boolean,
     "interface_language": fields.String,
@@ -23,7 +22,6 @@ account_with_role_fields = {
     "id": fields.String,
     "name": fields.String,
     "avatar": fields.String,
-    "avatar_url": AvatarUrlField,
     "email": fields.String,
     "last_login_at": TimestampField,
     "last_active_at": TimestampField,

api/fields/segment_fields.py

@@ -18,7 +18,6 @@ segment_fields = {
     "position": fields.Integer,
     "document_id": fields.String,
     "content": fields.String,
-    "sign_content": fields.String,
     "answer": fields.String,
     "word_count": fields.Integer,
     "tokens": fields.Integer,

api/libs/helper.py

@@ -43,18 +43,6 @@ class AppIconUrlField(fields.Raw):
         return None
 
 
-class AvatarUrlField(fields.Raw):
-    def output(self, key, obj):
-        if obj is None:
-            return None
-
-        from models.account import Account
-
-        if isinstance(obj, Account) and obj.avatar is not None:
-            return file_helpers.get_signed_file_url(obj.avatar)
-        return None
-
-
 class TimestampField(fields.Raw):
     def format(self, value) -> int:
         return int(value.timestamp())

api/models/dataset.py

@@ -13,7 +13,6 @@ from typing import Any, cast
 
 from sqlalchemy import func
 from sqlalchemy.dialects.postgresql import JSONB
-from sqlalchemy.orm import Mapped
 
 from configs import dify_config
 from core.rag.retrieval.retrieval_methods import RetrievalMethod
@@ -516,7 +515,7 @@ class DocumentSegment(db.Model):  # type: ignore[name-defined]
     tenant_id = db.Column(StringUUID, nullable=False)
     dataset_id = db.Column(StringUUID, nullable=False)
     document_id = db.Column(StringUUID, nullable=False)
-    position: Mapped[int]
+    position = db.Column(db.Integer, nullable=False)
     content = db.Column(db.Text, nullable=False)
     answer = db.Column(db.Text, nullable=True)
     word_count = db.Column(db.Integer, nullable=False)
@@ -584,10 +583,6 @@ class DocumentSegment(db.Model):  # type: ignore[name-defined]
         else:
             return []
 
-    @property
-    def sign_content(self):
-        return self.get_sign_content()
-
     def get_sign_content(self):
         signed_urls = []
         text = self.content
@@ -748,7 +743,7 @@ class DatasetKeywordTable(db.Model):  # type: ignore[name-defined]
                 if keyword_table_text:
                     return json.loads(keyword_table_text.decode("utf-8"), cls=SetDecoder)
                 return None
-            except Exception:
+            except Exception as e:
                 logging.exception(f"Failed to load keyword table from file: {file_key}")
                 return None
 

api/models/model.py

@@ -1486,8 +1486,9 @@ class ApiToken(Base):
     def generate_api_key(prefix, n):
         while True:
             result = prefix + generate_string(n)
-            if db.session.query(ApiToken).filter(ApiToken.token == result).count() > 0:
-                continue
+            while db.session.query(ApiToken).filter(ApiToken.token == result).count() > 0:
+                result = prefix + generate_string(n)
+
             return result
 
 

api/pyproject.toml

@@ -1,10 +1,9 @@
 [project]
 name = "dify-api"
 requires-python = ">=3.11,<3.13"
-dynamic = [ "dependencies" ]
 
 [build-system]
-requires = ["poetry-core>=2.0.0"]
+requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
@@ -49,7 +48,6 @@ numpy = "~1.26.4"
 oci = "~2.135.1"
 openai = "~1.52.0"
 openpyxl = "~3.1.5"
-opik = "~1.3.4"
 pandas = { version = "~2.2.2", extras = ["performance", "excel"] }
 pandas-stubs = "~2.2.3.241009"
 psycogreen = "~1.0.2"
@@ -159,4 +157,4 @@ pytest-mock = "~3.14.0"
 optional = true
 [tool.poetry.group.lint.dependencies]
 dotenv-linter = "~0.5.0"
-ruff = "~0.9.2"
+ruff = "~0.8.1"

api/pytest.ini

@@ -7,12 +7,6 @@ env =
     CODE_EXECUTION_API_KEY = dify-sandbox
     CODE_EXECUTION_ENDPOINT = http://127.0.0.1:8194
     CODE_MAX_STRING_LENGTH = 80000
-    PLUGIN_DAEMON_KEY=lYkiYYT6owG+71oLerGzA7GXCgOT++6ovaezWAjpCjf+Sjc3ZtU+qUEi
-    PLUGIN_DAEMON_URL=http://127.0.0.1:5002
-    PLUGIN_MAX_PACKAGE_SIZE=15728640
-    INNER_API_KEY_FOR_PLUGIN=QaHbTe77CtuXmsfyhR7+vRjI/+XbV1AaFy691iy+kGDv2Jvy0/eAh8Y1
-    MARKETPLACE_ENABLED=true
-    MARKETPLACE_API_URL=https://marketplace.dify.ai
     FIRECRAWL_API_KEY = fc-
     FIREWORKS_API_KEY = fw_aaaaaaaaaaaaaaaaaaaa
     GOOGLE_API_KEY = abcdefghijklmnopqrstuvwxyz

api/services/annotation_service.py

@@ -286,7 +286,7 @@ class AppAnnotationService:
             df = pd.read_csv(file)
             result = []
             for index, row in df.iterrows():
-                content = {"question": row.iloc[0], "answer": row.iloc[1]}
+                content = {"question": row[0], "answer": row[1]}
                 result.append(content)
             if len(result) == 0:
                 raise ValueError("The CSV file is empty.")

api/services/app_dsl_service.py

@@ -1,7 +1,7 @@
 import logging
 import uuid
 from enum import StrEnum
-from typing import Optional
+from typing import Optional, cast
 from urllib.parse import urlparse
 from uuid import uuid4
 
@@ -159,6 +159,15 @@ class AppDslService:
                         status=ImportStatus.FAILED,
                         error="Empty content from url",
                     )
+
+                try:
+                    content = cast(bytes, content).decode("utf-8")
+                except UnicodeDecodeError as e:
+                    return Import(
+                        id=import_id,
+                        status=ImportStatus.FAILED,
+                        error=f"Error decoding content: {e}",
+                    )
             except Exception as e:
                 return Import(
                     id=import_id,

api/services/audio_service.py

@@ -82,7 +82,7 @@ class AudioService:
         from app import app
         from extensions.ext_database import db
 
-        def invoke_tts(text_content: str, app_model: App, voice: Optional[str] = None):
+        def invoke_tts(text_content: str, app_model, voice: Optional[str] = None):
             with app.app_context():
                 if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
                     workflow = app_model.workflow
@@ -95,8 +95,6 @@ class AudioService:
 
                     voice = features_dict["text_to_speech"].get("voice") if voice is None else voice
                 else:
-                    if app_model.app_model_config is None:
-                        raise ValueError("AppModelConfig not found")
                     text_to_speech_dict = app_model.app_model_config.text_to_speech_dict
 
                     if not text_to_speech_dict.get("enabled"):

api/services/dataset_service.py

@@ -4,16 +4,13 @@ import logging
 import random
 import time
 import uuid
-from collections import Counter
 from typing import Any, Optional
 
 from flask_login import current_user  # type: ignore
 from sqlalchemy import func
-from sqlalchemy.orm import Session
 from werkzeug.exceptions import NotFound
 
 from configs import dify_config
-from core.entities import DEFAULT_PLUGIN_ID
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
@@ -74,7 +71,7 @@ from tasks.sync_website_document_indexing_task import sync_website_document_inde
 
 class DatasetService:
     @staticmethod
-    def get_datasets(page, per_page, tenant_id=None, user=None, search=None, tag_ids=None, include_all=False):
+    def get_datasets(page, per_page, tenant_id=None, user=None, search=None, tag_ids=None):
         query = Dataset.query.filter(Dataset.tenant_id == tenant_id).order_by(Dataset.created_at.desc())
 
         if user:
@@ -89,7 +86,7 @@ class DatasetService:
                 else:
                     return [], 0
             else:
-                if user.current_role != TenantAccountRole.OWNER or not include_all:
+                if user.current_role != TenantAccountRole.OWNER:
                     # show all datasets that the user has permission to access
                     if permitted_dataset_ids:
                         query = query.filter(
@@ -224,7 +221,8 @@ class DatasetService:
                 )
             except LLMBadRequestError:
                 raise ValueError(
-                    "No Embedding Model available. Please configure a valid provider in the Settings -> Model Provider."
+                    "No Embedding Model available. Please configure a valid provider "
+                    "in the Settings -> Model Provider."
                 )
             except ProviderTokenNotInitError as ex:
                 raise ValueError(f"The dataset in unavailable, due to: {ex.description}")
@@ -269,15 +267,7 @@ class DatasetService:
             external_knowledge_api_id = data.get("external_knowledge_api_id", None)
             if not external_knowledge_api_id:
                 raise ValueError("External knowledge api id is required.")
-
-            with Session(db.engine) as session:
-                external_knowledge_binding = (
-                    session.query(ExternalKnowledgeBindings).filter_by(dataset_id=dataset_id).first()
-                )
-
-                if not external_knowledge_binding:
-                    raise ValueError("External knowledge binding not found.")
-
+            external_knowledge_binding = ExternalKnowledgeBindings.query.filter_by(dataset_id=dataset_id).first()
             if (
                 external_knowledge_binding.external_knowledge_id != external_knowledge_id
                 or external_knowledge_binding.external_knowledge_api_id != external_knowledge_api_id
@@ -325,19 +315,8 @@ class DatasetService:
                     except ProviderTokenNotInitError as ex:
                         raise ValueError(ex.description)
             else:
-                # add default plugin id to both setting sets, to make sure the plugin model provider is consistent
-                plugin_model_provider = dataset.embedding_model_provider
-                if "/" not in plugin_model_provider:
-                    plugin_model_provider = f"{DEFAULT_PLUGIN_ID}/{plugin_model_provider}/{plugin_model_provider}"
-
-                new_plugin_model_provider = data["embedding_model_provider"]
-                if "/" not in new_plugin_model_provider:
-                    new_plugin_model_provider = (
-                        f"{DEFAULT_PLUGIN_ID}/{new_plugin_model_provider}/{new_plugin_model_provider}"
-                    )
-
                 if (
-                    new_plugin_model_provider != plugin_model_provider
+                    data["embedding_model_provider"] != dataset.embedding_model_provider
                     or data["embedding_model"] != dataset.embedding_model
                 ):
                     action = "update"
@@ -880,7 +859,7 @@ class DocumentService:
                 position = DocumentService.get_documents_position(dataset.id)
                 document_ids = []
                 duplicate_document_ids = []
-                if knowledge_config.data_source.info_list.data_source_type == "upload_file":  # type: ignore
+                if knowledge_config.data_source.info_list.data_source_type == "upload_file":
                     upload_file_list = knowledge_config.data_source.info_list.file_info_list.file_ids  # type: ignore
                     for file_id in upload_file_list:
                         file = (
@@ -922,7 +901,7 @@ class DocumentService:
                         document = DocumentService.build_document(
                             dataset,
                             dataset_process_rule.id,  # type: ignore
-                            knowledge_config.data_source.info_list.data_source_type,  # type: ignore
+                            knowledge_config.data_source.info_list.data_source_type,
                             knowledge_config.doc_form,
                             knowledge_config.doc_language,
                             data_source_info,
@@ -937,8 +916,8 @@ class DocumentService:
                         document_ids.append(document.id)
                         documents.append(document)
                         position += 1
-                elif knowledge_config.data_source.info_list.data_source_type == "notion_import":  # type: ignore
-                    notion_info_list = knowledge_config.data_source.info_list.notion_info_list  # type: ignore
+                elif knowledge_config.data_source.info_list.data_source_type == "notion_import":
+                    notion_info_list = knowledge_config.data_source.info_list.notion_info_list
                     if not notion_info_list:
                         raise ValueError("No notion info list found.")
                     exist_page_ids = []
@@ -977,7 +956,7 @@ class DocumentService:
                                 document = DocumentService.build_document(
                                     dataset,
                                     dataset_process_rule.id,  # type: ignore
-                                    knowledge_config.data_source.info_list.data_source_type,  # type: ignore
+                                    knowledge_config.data_source.info_list.data_source_type,
                                     knowledge_config.doc_form,
                                     knowledge_config.doc_language,
                                     data_source_info,
@@ -997,8 +976,8 @@ class DocumentService:
                     # delete not selected documents
                     if len(exist_document) > 0:
                         clean_notion_document_task.delay(list(exist_document.values()), dataset.id)
-                elif knowledge_config.data_source.info_list.data_source_type == "website_crawl":  # type: ignore
-                    website_info = knowledge_config.data_source.info_list.website_info_list  # type: ignore
+                elif knowledge_config.data_source.info_list.data_source_type == "website_crawl":
+                    website_info = knowledge_config.data_source.info_list.website_info_list
                     if not website_info:
                         raise ValueError("No website info list found.")
                     urls = website_info.urls
@@ -1017,7 +996,7 @@ class DocumentService:
                         document = DocumentService.build_document(
                             dataset,
                             dataset_process_rule.id,  # type: ignore
-                            knowledge_config.data_source.info_list.data_source_type,  # type: ignore
+                            knowledge_config.data_source.info_list.data_source_type,
                             knowledge_config.doc_form,
                             knowledge_config.doc_language,
                             data_source_info,
@@ -1216,20 +1195,20 @@ class DocumentService:
 
         if features.billing.enabled:
             count = 0
-            if knowledge_config.data_source.info_list.data_source_type == "upload_file":  # type: ignore
+            if knowledge_config.data_source.info_list.data_source_type == "upload_file":
                 upload_file_list = (
-                    knowledge_config.data_source.info_list.file_info_list.file_ids  # type: ignore
-                    if knowledge_config.data_source.info_list.file_info_list  # type: ignore
+                    knowledge_config.data_source.info_list.file_info_list.file_ids
+                    if knowledge_config.data_source.info_list.file_info_list
                     else []
                 )
                 count = len(upload_file_list)
-            elif knowledge_config.data_source.info_list.data_source_type == "notion_import":  # type: ignore
-                notion_info_list = knowledge_config.data_source.info_list.notion_info_list  # type: ignore
+            elif knowledge_config.data_source.info_list.data_source_type == "notion_import":
+                notion_info_list = knowledge_config.data_source.info_list.notion_info_list
                 if notion_info_list:
                     for notion_info in notion_info_list:
                         count = count + len(notion_info.pages)
-            elif knowledge_config.data_source.info_list.data_source_type == "website_crawl":  # type: ignore
-                website_info = knowledge_config.data_source.info_list.website_info_list  # type: ignore
+            elif knowledge_config.data_source.info_list.data_source_type == "website_crawl":
+                website_info = knowledge_config.data_source.info_list.website_info_list
                 if website_info:
                     count = len(website_info.urls)
             batch_upload_limit = int(dify_config.BATCH_UPLOAD_LIMIT)
@@ -1260,7 +1239,7 @@ class DocumentService:
         dataset = Dataset(
             tenant_id=tenant_id,
             name="",
-            data_source_type=knowledge_config.data_source.info_list.data_source_type,  # type: ignore
+            data_source_type=knowledge_config.data_source.info_list.data_source_type,
             indexing_technique=knowledge_config.indexing_technique,
             created_by=account.id,
             embedding_model=knowledge_config.embedding_model,
@@ -1635,11 +1614,8 @@ class SegmentService:
                     segment.answer = args.answer
                     segment.word_count += len(args.answer) if args.answer else 0
                 word_count_change = segment.word_count - word_count_change
-                keyword_changed = False
                 if args.keywords:
-                    if Counter(segment.keywords) != Counter(args.keywords):
-                        segment.keywords = args.keywords
-                        keyword_changed = True
+                    segment.keywords = args.keywords
                 segment.enabled = True
                 segment.disabled_at = None
                 segment.disabled_by = None
@@ -1650,6 +1626,13 @@ class SegmentService:
                     document.word_count = max(0, document.word_count + word_count_change)
                     db.session.add(document)
                 # update segment index task
+                if args.enabled:
+                    VectorService.create_segments_vector(
+                        [args.keywords] if args.keywords else None,
+                        [segment],
+                        dataset,
+                        document.doc_form,
+                    )
                 if document.doc_form == IndexType.PARENT_CHILD_INDEX and args.regenerate_child_chunks:
                     # regenerate child chunks
                     # get embedding model instance
@@ -1682,14 +1665,6 @@ class SegmentService:
                     VectorService.generate_child_chunks(
                         segment, document, dataset, embedding_model_instance, processing_rule, True
                     )
-                elif document.doc_form in (IndexType.PARAGRAPH_INDEX, IndexType.QA_INDEX):
-                    if args.enabled or keyword_changed:
-                        VectorService.create_segments_vector(
-                            [args.keywords] if args.keywords else None,
-                            [segment],
-                            dataset,
-                            document.doc_form,
-                        )
             else:
                 segment_hash = helper.generate_text_hash(content)
                 tokens = 0

api/services/entities/knowledge_entities/knowledge_entities.py

@@ -97,7 +97,7 @@ class KnowledgeConfig(BaseModel):
     original_document_id: Optional[str] = None
     duplicate: bool = True
     indexing_technique: Literal["high_quality", "economy"]
-    data_source: Optional[DataSource] = None
+    data_source: DataSource
     process_rule: Optional[ProcessRule] = None
     retrieval_model: Optional[RetrievalModel] = None
     doc_form: str = "text_model"

api/services/external_knowledge_service.py

@@ -155,7 +155,7 @@ class ExternalDatasetService:
             if custom_parameters:
                 for parameter in custom_parameters:
                     if parameter.get("required", False) and not process_parameter.get(parameter.get("name")):
-                        raise ValueError(f"{parameter.get('name')} is required")
+                        raise ValueError(f'{parameter.get("name")} is required')
 
     @staticmethod
     def process_external_api(

api/services/ops_service.py

@@ -59,15 +59,6 @@ class OpsService:
             except Exception:
                 new_decrypt_tracing_config.update({"project_url": "https://smith.langchain.com/"})
 
-        if tracing_provider == "opik" and (
-            "project_url" not in decrypt_tracing_config or not decrypt_tracing_config.get("project_url")
-        ):
-            try:
-                project_url = OpsTraceManager.get_trace_config_project_url(decrypt_tracing_config, tracing_provider)
-                new_decrypt_tracing_config.update({"project_url": project_url})
-            except Exception:
-                new_decrypt_tracing_config.update({"project_url": "https://www.comet.com/opik/"})
-
         trace_config_data.tracing_config = new_decrypt_tracing_config
         return trace_config_data.to_dict()
 
@@ -101,7 +92,7 @@ class OpsService:
         if tracing_provider == "langfuse":
             project_key = OpsTraceManager.get_trace_config_project_key(tracing_config, tracing_provider)
             project_url = "{host}/project/{key}".format(host=tracing_config.get("host"), key=project_key)
-        elif tracing_provider in ("langsmith", "opik"):
+        elif tracing_provider == "langsmith":
             project_url = OpsTraceManager.get_trace_config_project_url(tracing_config, tracing_provider)
         else:
             project_url = None

api/tasks/batch_create_segment_to_index_task.py

@@ -5,8 +5,7 @@ import uuid
 
 import click
 from celery import shared_task  # type: ignore
-from sqlalchemy import func, select
-from sqlalchemy.orm import Session
+from sqlalchemy import func
 
 from core.model_manager import ModelManager
 from core.model_runtime.entities.model_entities import ModelType
@@ -19,12 +18,7 @@ from services.vector_service import VectorService
 
 @shared_task(queue="dataset")
 def batch_create_segment_to_index_task(
-    job_id: str,
-    content: list,
-    dataset_id: str,
-    document_id: str,
-    tenant_id: str,
-    user_id: str,
+    job_id: str, content: list, dataset_id: str, document_id: str, tenant_id: str, user_id: str
 ):
     """
     Async batch create segment to index
@@ -43,35 +37,25 @@ def batch_create_segment_to_index_task(
     indexing_cache_key = "segment_batch_import_{}".format(job_id)
 
     try:
-        with Session(db.engine) as session:
-            dataset = session.get(Dataset, dataset_id)
-            if not dataset:
-                raise ValueError("Dataset not exist.")
+        dataset = db.session.query(Dataset).filter(Dataset.id == dataset_id).first()
+        if not dataset:
+            raise ValueError("Dataset not exist.")
 
-            dataset_document = session.get(Document, document_id)
-            if not dataset_document:
-                raise ValueError("Document not exist.")
+        dataset_document = db.session.query(Document).filter(Document.id == document_id).first()
+        if not dataset_document:
+            raise ValueError("Document not exist.")
 
-            if (
-                not dataset_document.enabled
-                or dataset_document.archived
-                or dataset_document.indexing_status != "completed"
-            ):
-                raise ValueError("Document is not available.")
-            document_segments = []
-            embedding_model = None
-            if dataset.indexing_technique == "high_quality":
-                model_manager = ModelManager()
-                embedding_model = model_manager.get_model_instance(
-                    tenant_id=dataset.tenant_id,
-                    provider=dataset.embedding_model_provider,
-                    model_type=ModelType.TEXT_EMBEDDING,
-                    model=dataset.embedding_model,
-                )
-            word_count_change = 0
-            segments_to_insert: list[str] = []
-            max_position_stmt = select(func.max(DocumentSegment.position)).where(
-                DocumentSegment.document_id == dataset_document.id
+        if not dataset_document.enabled or dataset_document.archived or dataset_document.indexing_status != "completed":
+            raise ValueError("Document is not available.")
+        document_segments = []
+        embedding_model = None
+        if dataset.indexing_technique == "high_quality":
+            model_manager = ModelManager()
+            embedding_model = model_manager.get_model_instance(
+                tenant_id=dataset.tenant_id,
+                provider=dataset.embedding_model_provider,
+                model_type=ModelType.TEXT_EMBEDDING,
+                model=dataset.embedding_model,
             )
         word_count_change = 0
         if embedding_model:
@@ -119,10 +103,7 @@ def batch_create_segment_to_index_task(
         redis_client.setex(indexing_cache_key, 600, "completed")
         end_at = time.perf_counter()
         logging.info(
-            click.style(
-                "Segment batch created job: {} latency: {}".format(job_id, end_at - start_at),
-                fg="green",
-            )
+            click.style("Segment batch created job: {} latency: {}".format(job_id, end_at - start_at), fg="green")
         )
     except Exception:
         logging.exception("Segments batch created index failed")

api/tests/artifact_tests/dependencies/test_dependencies_sorted.py

@@ -44,6 +44,6 @@ def test_duplicated_dependency_crossing_groups() -> None:
         dependency_names = list(dependencies.keys())
         all_dependency_names.extend(dependency_names)
     expected_all_dependency_names = set(all_dependency_names)
-    assert sorted(expected_all_dependency_names) == sorted(all_dependency_names), (
-        "Duplicated dependencies crossing groups are found"
-    )
+    assert sorted(expected_all_dependency_names) == sorted(
+        all_dependency_names
+    ), "Duplicated dependencies crossing groups are found"

api/tests/integration_tests/.env.example

@@ -85,7 +85,7 @@ VOLC_EMBEDDING_ENDPOINT_ID=
 ZHINAO_API_KEY=
 
 # Plugin configuration
-PLUGIN_DAEMON_KEY=
+PLUGIN_API_KEY=
 PLUGIN_DAEMON_URL=
 INNER_API_KEY=
 

api/tests/integration_tests/controllers/test_controllers.py

@@ -4,6 +4,7 @@ from app_fixture import mock_user  # type: ignore
 
 
 def test_post_requires_login(app):
-    with app.test_client() as client, patch("flask_login.utils._get_user", mock_user):
-        response = client.get("/console/api/data-source/integrates")
-        assert response.status_code == 200
+    with app.test_client() as client:
+        with patch("flask_login.utils._get_user", mock_user):
+            response = client.get("/console/api/data-source/integrates")
+            assert response.status_code == 200

api/tests/integration_tests/model_runtime/__mock/plugin_daemon.py

@@ -1,44 +0,0 @@
-import os
-from collections.abc import Callable
-
-import pytest
-
-# import monkeypatch
-from _pytest.monkeypatch import MonkeyPatch
-
-from core.plugin.manager.model import PluginModelManager
-from tests.integration_tests.model_runtime.__mock.plugin_model import MockModelClass
-
-
-def mock_plugin_daemon(
-    monkeypatch: MonkeyPatch,
-) -> Callable[[], None]:
-    """
-    mock openai module
-
-    :param monkeypatch: pytest monkeypatch fixture
-    :return: unpatch function
-    """
-
-    def unpatch() -> None:
-        monkeypatch.undo()
-
-    monkeypatch.setattr(PluginModelManager, "invoke_llm", MockModelClass.invoke_llm)
-    monkeypatch.setattr(PluginModelManager, "fetch_model_providers", MockModelClass.fetch_model_providers)
-    monkeypatch.setattr(PluginModelManager, "get_model_schema", MockModelClass.get_model_schema)
-
-    return unpatch
-
-
-MOCK = os.getenv("MOCK_SWITCH", "false").lower() == "true"
-
-
-@pytest.fixture
-def setup_model_mock(monkeypatch):
-    if MOCK:
-        unpatch = mock_plugin_daemon(monkeypatch)
-
-    yield
-
-    if MOCK:
-        unpatch()

api/tests/integration_tests/model_runtime/__mock/plugin_model.py

@@ -1,249 +0,0 @@
-import datetime
-import uuid
-from collections.abc import Generator, Sequence
-from decimal import Decimal
-from json import dumps
-
-# import monkeypatch
-from typing import Optional
-
-from core.model_runtime.entities.common_entities import I18nObject
-from core.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
-from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, PromptMessageTool
-from core.model_runtime.entities.model_entities import (
-    AIModelEntity,
-    FetchFrom,
-    ModelFeature,
-    ModelPropertyKey,
-    ModelType,
-)
-from core.model_runtime.entities.provider_entities import ConfigurateMethod, ProviderEntity
-from core.plugin.entities.plugin_daemon import PluginModelProviderEntity
-from core.plugin.manager.model import PluginModelManager
-
-
-class MockModelClass(PluginModelManager):
-    def fetch_model_providers(self, tenant_id: str) -> Sequence[PluginModelProviderEntity]:
-        """
-        Fetch model providers for the given tenant.
-        """
-        return [
-            PluginModelProviderEntity(
-                id=uuid.uuid4().hex,
-                created_at=datetime.datetime.now(),
-                updated_at=datetime.datetime.now(),
-                provider="openai",
-                tenant_id=tenant_id,
-                plugin_unique_identifier="langgenius/openai/openai",
-                plugin_id="langgenius/openai",
-                declaration=ProviderEntity(
-                    provider="openai",
-                    label=I18nObject(
-                        en_US="OpenAI",
-                        zh_Hans="OpenAI",
-                    ),
-                    description=I18nObject(
-                        en_US="OpenAI",
-                        zh_Hans="OpenAI",
-                    ),
-                    icon_small=I18nObject(
-                        en_US="https://example.com/icon_small.png",
-                        zh_Hans="https://example.com/icon_small.png",
-                    ),
-                    icon_large=I18nObject(
-                        en_US="https://example.com/icon_large.png",
-                        zh_Hans="https://example.com/icon_large.png",
-                    ),
-                    supported_model_types=[ModelType.LLM],
-                    configurate_methods=[ConfigurateMethod.PREDEFINED_MODEL],
-                    models=[
-                        AIModelEntity(
-                            model="gpt-3.5-turbo",
-                            label=I18nObject(
-                                en_US="gpt-3.5-turbo",
-                                zh_Hans="gpt-3.5-turbo",
-                            ),
-                            model_type=ModelType.LLM,
-                            fetch_from=FetchFrom.PREDEFINED_MODEL,
-                            model_properties={},
-                            features=[ModelFeature.TOOL_CALL, ModelFeature.MULTI_TOOL_CALL],
-                        ),
-                        AIModelEntity(
-                            model="gpt-3.5-turbo-instruct",
-                            label=I18nObject(
-                                en_US="gpt-3.5-turbo-instruct",
-                                zh_Hans="gpt-3.5-turbo-instruct",
-                            ),
-                            model_type=ModelType.LLM,
-                            fetch_from=FetchFrom.PREDEFINED_MODEL,
-                            model_properties={
-                                ModelPropertyKey.MODE: LLMMode.COMPLETION,
-                            },
-                            features=[],
-                        ),
-                    ],
-                ),
-            )
-        ]
-
-    def get_model_schema(
-        self,
-        tenant_id: str,
-        user_id: str,
-        plugin_id: str,
-        provider: str,
-        model_type: str,
-        model: str,
-        credentials: dict,
-    ) -> AIModelEntity | None:
-        """
-        Get model schema
-        """
-        return AIModelEntity(
-            model=model,
-            label=I18nObject(
-                en_US="OpenAI",
-                zh_Hans="OpenAI",
-            ),
-            model_type=ModelType(model_type),
-            fetch_from=FetchFrom.PREDEFINED_MODEL,
-            model_properties={},
-            features=[ModelFeature.TOOL_CALL, ModelFeature.MULTI_TOOL_CALL] if model == "gpt-3.5-turbo" else [],
-        )
-
-    @staticmethod
-    def generate_function_call(
-        tools: Optional[list[PromptMessageTool]],
-    ) -> Optional[AssistantPromptMessage.ToolCall]:
-        if not tools or len(tools) == 0:
-            return None
-        function: PromptMessageTool = tools[0]
-        function_name = function.name
-        function_parameters = function.parameters
-        function_parameters_type = function_parameters["type"]
-        if function_parameters_type != "object":
-            return None
-        function_parameters_properties = function_parameters["properties"]
-        function_parameters_required = function_parameters["required"]
-        parameters = {}
-        for parameter_name, parameter in function_parameters_properties.items():
-            if parameter_name not in function_parameters_required:
-                continue
-            parameter_type = parameter["type"]
-            if parameter_type == "string":
-                if "enum" in parameter:
-                    if len(parameter["enum"]) == 0:
-                        continue
-                    parameters[parameter_name] = parameter["enum"][0]
-                else:
-                    parameters[parameter_name] = "kawaii"
-            elif parameter_type == "integer":
-                parameters[parameter_name] = 114514
-            elif parameter_type == "number":
-                parameters[parameter_name] = 1919810.0
-            elif parameter_type == "boolean":
-                parameters[parameter_name] = True
-
-        return AssistantPromptMessage.ToolCall(
-            id=str(uuid.uuid4()),
-            type="function",
-            function=AssistantPromptMessage.ToolCall.ToolCallFunction(
-                name=function_name,
-                arguments=dumps(parameters),
-            ),
-        )
-
-    @staticmethod
-    def mocked_chat_create_sync(
-        model: str,
-        prompt_messages: list[PromptMessage],
-        tools: Optional[list[PromptMessageTool]] = None,
-    ) -> LLMResult:
-        tool_call = MockModelClass.generate_function_call(tools=tools)
-
-        return LLMResult(
-            id=str(uuid.uuid4()),
-            model=model,
-            prompt_messages=prompt_messages,
-            message=AssistantPromptMessage(content="elaina", tool_calls=[tool_call] if tool_call else []),
-            usage=LLMUsage(
-                prompt_tokens=2,
-                completion_tokens=1,
-                total_tokens=3,
-                prompt_unit_price=Decimal(0.0001),
-                completion_unit_price=Decimal(0.0002),
-                prompt_price_unit=Decimal(1),
-                prompt_price=Decimal(0.0001),
-                completion_price_unit=Decimal(1),
-                completion_price=Decimal(0.0002),
-                total_price=Decimal(0.0003),
-                currency="USD",
-                latency=0.001,
-            ),
-        )
-
-    @staticmethod
-    def mocked_chat_create_stream(
-        model: str,
-        prompt_messages: list[PromptMessage],
-        tools: Optional[list[PromptMessageTool]] = None,
-    ) -> Generator[LLMResultChunk, None, None]:
-        tool_call = MockModelClass.generate_function_call(tools=tools)
-
-        full_text = "Hello, world!\n\n```python\nprint('Hello, world!')\n```"
-        for i in range(0, len(full_text) + 1):
-            if i == len(full_text):
-                yield LLMResultChunk(
-                    model=model,
-                    prompt_messages=prompt_messages,
-                    delta=LLMResultChunkDelta(
-                        index=0,
-                        message=AssistantPromptMessage(
-                            content="",
-                            tool_calls=[tool_call] if tool_call else [],
-                        ),
-                    ),
-                )
-            else:
-                yield LLMResultChunk(
-                    model=model,
-                    prompt_messages=prompt_messages,
-                    delta=LLMResultChunkDelta(
-                        index=0,
-                        message=AssistantPromptMessage(
-                            content=full_text[i],
-                            tool_calls=[tool_call] if tool_call else [],
-                        ),
-                        usage=LLMUsage(
-                            prompt_tokens=2,
-                            completion_tokens=17,
-                            total_tokens=19,
-                            prompt_unit_price=Decimal(0.0001),
-                            completion_unit_price=Decimal(0.0002),
-                            prompt_price_unit=Decimal(1),
-                            prompt_price=Decimal(0.0001),
-                            completion_price_unit=Decimal(1),
-                            completion_price=Decimal(0.0002),
-                            total_price=Decimal(0.0003),
-                            currency="USD",
-                            latency=0.001,
-                        ),
-                    ),
-                )
-
-    def invoke_llm(
-        self: PluginModelManager,
-        *,
-        tenant_id: str,
-        user_id: str,
-        plugin_id: str,
-        provider: str,
-        model: str,
-        credentials: dict,
-        prompt_messages: list[PromptMessage],
-        model_parameters: Optional[dict] = None,
-        tools: Optional[list[PromptMessageTool]] = None,
-        stop: Optional[list[str]] = None,
-        stream: bool = True,
-    ):
-        return MockModelClass.mocked_chat_create_stream(model=model, prompt_messages=prompt_messages, tools=tools)

api/tests/integration_tests/model_runtime/gpustack/test_speech2text.py

@@ -0,0 +1,55 @@
+import os
+from pathlib import Path
+
+import pytest
+
+from core.model_runtime.errors.validate import CredentialsValidateFailedError
+from core.model_runtime.model_providers.gpustack.speech2text.speech2text import GPUStackSpeech2TextModel
+
+
+def test_validate_credentials():
+    model = GPUStackSpeech2TextModel()
+
+    with pytest.raises(CredentialsValidateFailedError):
+        model.validate_credentials(
+            model="faster-whisper-medium",
+            credentials={
+                "endpoint_url": "invalid_url",
+                "api_key": "invalid_api_key",
+            },
+        )
+
+    model.validate_credentials(
+        model="faster-whisper-medium",
+        credentials={
+            "endpoint_url": os.environ.get("GPUSTACK_SERVER_URL"),
+            "api_key": os.environ.get("GPUSTACK_API_KEY"),
+        },
+    )
+
+
+def test_invoke_model():
+    model = GPUStackSpeech2TextModel()
+
+    # Get the directory of the current file
+    current_dir = os.path.dirname(os.path.abspath(__file__))
+
+    # Get assets directory
+    assets_dir = os.path.join(os.path.dirname(current_dir), "assets")
+
+    # Construct the path to the audio file
+    audio_file_path = os.path.join(assets_dir, "audio.mp3")
+
+    file = Path(audio_file_path).read_bytes()
+
+    result = model.invoke(
+        model="faster-whisper-medium",
+        credentials={
+            "endpoint_url": os.environ.get("GPUSTACK_SERVER_URL"),
+            "api_key": os.environ.get("GPUSTACK_API_KEY"),
+        },
+        file=file,
+    )
+
+    assert isinstance(result, str)
+    assert result == "1, 2, 3, 4, 5, 6, 7, 8, 9, 10"

api/tests/integration_tests/model_runtime/gpustack/test_tts.py

@@ -0,0 +1,24 @@
+import os
+
+from core.model_runtime.model_providers.gpustack.tts.tts import GPUStackText2SpeechModel
+
+
+def test_invoke_model():
+    model = GPUStackText2SpeechModel()
+
+    result = model.invoke(
+        model="cosyvoice-300m-sft",
+        tenant_id="test",
+        credentials={
+            "endpoint_url": os.environ.get("GPUSTACK_SERVER_URL"),
+            "api_key": os.environ.get("GPUSTACK_API_KEY"),
+        },
+        content_text="Hello world",
+        voice="Chinese Female",
+    )
+
+    content = b""
+    for chunk in result:
+        content += chunk
+
+    assert content != b""