test: enhance text generation and workflow callbacks tests

- Added comprehensive tests for the useTextGeneration hook, covering handleSend and handleStop functionalities, including file processing and error handling.
- Improved workflow callbacks tests to handle edge cases and ensure graceful handling of missing nodes and undefined tracing.
- Introduced new test cases for validating prompt variables and processing files in the text generation flow.

.codex/skills/component-refactoring

@@ -1 +0,0 @@
-../../.agents/skills/component-refactoring

.codex/skills/frontend-code-review

@@ -1 +0,0 @@
-../../.agents/skills/frontend-code-review

.codex/skills/frontend-testing

@@ -1 +0,0 @@
-../../.agents/skills/frontend-testing

.codex/skills/orpc-contract-first

@@ -1 +0,0 @@
-../../.agents/skills/orpc-contract-first

.github/CODEOWNERS

@@ -9,6 +9,9 @@
 # CODEOWNERS file
 /.github/CODEOWNERS @laipz8200 @crazywoola
 
+# Agents
+/.agents/skills/ @hyoban
+
 # Docs
 /docs/ @crazywoola
 
@@ -21,6 +24,10 @@
 /api/services/tools/mcp_tools_manage_service.py @Nov1c444
 /api/controllers/mcp/ @Nov1c444
 /api/controllers/console/app/mcp_server.py @Nov1c444
+
+# Backend - Tests
+/api/tests/ @laipz8200 @QuantumGhost
+
 /api/tests/**/*mcp* @Nov1c444
 
 # Backend - Workflow - Engine (Core graph execution engine)
@@ -231,6 +238,9 @@
 # Frontend - Base Components
 /web/app/components/base/ @iamjoel @zxhlyh
 
+# Frontend - Base Components Tests
+/web/app/components/base/**/*.spec.tsx @hyoban @CodingOnStar
+
 # Frontend - Utils and Hooks
 /web/utils/classnames.ts @iamjoel @zxhlyh
 /web/utils/time.ts @iamjoel @zxhlyh

.github/workflows/autofix.yml

@@ -79,29 +79,6 @@ jobs:
           find . -name "*.py" -type f -exec sed -i.bak -E 's/"([^"]+)" \| None/Optional["\1"]/g; s/'"'"'([^'"'"']+)'"'"' \| None/Optional['"'"'\1'"'"']/g' {} \;
           find . -name "*.py.bak" -type f -delete
 
-      - name: Install pnpm
-        uses: pnpm/action-setup@v4
-        with:
-          package_json_file: web/package.json
-          run_install: false
-
-      - name: Setup Node.js
-        uses: actions/setup-node@v6
-        with:
-          node-version: 24
-          cache: pnpm
-          cache-dependency-path: ./web/pnpm-lock.yaml
-
-      - name: Install web dependencies
-        run: |
-          cd web
-          pnpm install --frozen-lockfile
-
-      - name: ESLint autofix
-        run: |
-          cd web
-          pnpm lint:fix || true
-
       # mdformat breaks YAML front matter in markdown files. Add --exclude for directories containing YAML front matter.
       - name: mdformat
         run: |

.github/workflows/deploy-hitl.yml

@@ -4,8 +4,7 @@ on:
   workflow_run:
     workflows: ["Build and Push API & Web"]
     branches:
-      - "feat/hitl-frontend"
-      - "feat/hitl-backend"
+      - "build/feat/hitl"
     types:
       - completed
 
@@ -14,10 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     if: |
       github.event.workflow_run.conclusion == 'success' &&
-      (
-        github.event.workflow_run.head_branch == 'feat/hitl-frontend' ||
-        github.event.workflow_run.head_branch == 'feat/hitl-backend'
-      )
+      github.event.workflow_run.head_branch == 'build/feat/hitl'
     steps:
       - name: Deploy to server
         uses: appleboy/ssh-action@v1

.github/workflows/web-tests.yml

@@ -39,7 +39,7 @@ jobs:
         run: pnpm install --frozen-lockfile
 
       - name: Run tests
-        run: pnpm test:coverage
+        run: pnpm test:ci
 
       - name: Coverage Summary
         if: always()

api/.importlinter

@@ -136,7 +136,6 @@ ignore_imports =
     core.workflow.nodes.llm.llm_utils -> models.provider
     core.workflow.nodes.llm.llm_utils -> services.credit_pool_service
     core.workflow.nodes.llm.node -> core.tools.signature
-    core.workflow.nodes.template_transform.template_transform_node -> configs
     core.workflow.nodes.tool.tool_node -> core.callback_handler.workflow_tool_callback_handler
     core.workflow.nodes.tool.tool_node -> core.tools.tool_engine
     core.workflow.nodes.tool.tool_node -> core.tools.tool_manager

api/README.md

@@ -122,7 +122,7 @@ These commands assume you start from the repository root.
 
    ```bash
    cd api
-   uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention
+   uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q api_token,dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention
    ```
 
 1. Optional: start Celery Beat (scheduled tasks, in a new terminal).

api/commands.py

@@ -739,8 +739,10 @@ def upgrade_db():
 
             click.echo(click.style("Database migration successful!", fg="green"))
 
-        except Exception:
+        except Exception as e:
             logger.exception("Failed to execute database migration")
+            click.echo(click.style(f"Database migration failed: {e}", fg="red"))
+            raise SystemExit(1)
         finally:
             lock.release()
     else:
@@ -1450,54 +1452,58 @@ def clear_orphaned_file_records(force: bool):
         all_ids_in_tables = []
         for ids_table in ids_tables:
             query = ""
-            if ids_table["type"] == "uuid":
-                click.echo(
-                    click.style(
-                        f"- Listing file ids in column {ids_table['column']} in table {ids_table['table']}", fg="white"
+            match ids_table["type"]:
+                case "uuid":
+                    click.echo(
+                        click.style(
+                            f"- Listing file ids in column {ids_table['column']} in table {ids_table['table']}",
+                            fg="white",
+                        )
                     )
-                )
-                query = (
-                    f"SELECT {ids_table['column']} FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
-                )
-                with db.engine.begin() as conn:
-                    rs = conn.execute(sa.text(query))
-                for i in rs:
-                    all_ids_in_tables.append({"table": ids_table["table"], "id": str(i[0])})
-            elif ids_table["type"] == "text":
-                click.echo(
-                    click.style(
-                        f"- Listing file-id-like strings in column {ids_table['column']} in table {ids_table['table']}",
-                        fg="white",
+                    c = ids_table["column"]
+                    query = f"SELECT {c} FROM {ids_table['table']} WHERE {c} IS NOT NULL"
+                    with db.engine.begin() as conn:
+                        rs = conn.execute(sa.text(query))
+                    for i in rs:
+                        all_ids_in_tables.append({"table": ids_table["table"], "id": str(i[0])})
+                case "text":
+                    t = ids_table["table"]
+                    click.echo(
+                        click.style(
+                            f"- Listing file-id-like strings in column {ids_table['column']} in table {t}",
+                            fg="white",
+                        )
                     )
-                )
-                query = (
-                    f"SELECT regexp_matches({ids_table['column']}, '{guid_regexp}', 'g') AS extracted_id "
-                    f"FROM {ids_table['table']}"
-                )
-                with db.engine.begin() as conn:
-                    rs = conn.execute(sa.text(query))
-                for i in rs:
-                    for j in i[0]:
-                        all_ids_in_tables.append({"table": ids_table["table"], "id": j})
-            elif ids_table["type"] == "json":
-                click.echo(
-                    click.style(
-                        (
-                            f"- Listing file-id-like JSON string in column {ids_table['column']} "
-                            f"in table {ids_table['table']}"
-                        ),
-                        fg="white",
+                    query = (
+                        f"SELECT regexp_matches({ids_table['column']}, '{guid_regexp}', 'g') AS extracted_id "
+                        f"FROM {ids_table['table']}"
                     )
-                )
-                query = (
-                    f"SELECT regexp_matches({ids_table['column']}::text, '{guid_regexp}', 'g') AS extracted_id "
-                    f"FROM {ids_table['table']}"
-                )
-                with db.engine.begin() as conn:
-                    rs = conn.execute(sa.text(query))
-                for i in rs:
-                    for j in i[0]:
-                        all_ids_in_tables.append({"table": ids_table["table"], "id": j})
+                    with db.engine.begin() as conn:
+                        rs = conn.execute(sa.text(query))
+                    for i in rs:
+                        for j in i[0]:
+                            all_ids_in_tables.append({"table": ids_table["table"], "id": j})
+                case "json":
+                    click.echo(
+                        click.style(
+                            (
+                                f"- Listing file-id-like JSON string in column {ids_table['column']} "
+                                f"in table {ids_table['table']}"
+                            ),
+                            fg="white",
+                        )
+                    )
+                    query = (
+                        f"SELECT regexp_matches({ids_table['column']}::text, '{guid_regexp}', 'g') AS extracted_id "
+                        f"FROM {ids_table['table']}"
+                    )
+                    with db.engine.begin() as conn:
+                        rs = conn.execute(sa.text(query))
+                    for i in rs:
+                        for j in i[0]:
+                            all_ids_in_tables.append({"table": ids_table["table"], "id": j})
+                case _:
+                    pass
         click.echo(click.style(f"Found {len(all_ids_in_tables)} file ids in tables.", fg="white"))
 
     except Exception as e:
@@ -1737,59 +1743,18 @@ def file_usage(
                 if src_filter != src:
                     continue
 
-        if ids_table["type"] == "uuid":
-            # Direct UUID match
-            query = (
-                f"SELECT {ids_table['pk_column']}, {ids_table['column']} "
-                f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
-            )
-            with db.engine.begin() as conn:
-                rs = conn.execute(sa.text(query))
-                for row in rs:
-                    record_id = str(row[0])
-                    ref_file_id = str(row[1])
-                    if ref_file_id not in file_key_map:
-                        continue
-                    storage_key = file_key_map[ref_file_id]
-
-                    # Apply filters
-                    if file_id and ref_file_id != file_id:
-                        continue
-                    if key and not storage_key.endswith(key):
-                        continue
-
-                    # Only collect items within the requested page range
-                    if offset <= total_count < offset + limit:
-                        paginated_usages.append(
-                            {
-                                "src": f"{ids_table['table']}.{ids_table['column']}",
-                                "record_id": record_id,
-                                "file_id": ref_file_id,
-                                "key": storage_key,
-                            }
-                        )
-                    total_count += 1
-
-        elif ids_table["type"] in ("text", "json"):
-            # Extract UUIDs from text/json content
-            column_cast = f"{ids_table['column']}::text" if ids_table["type"] == "json" else ids_table["column"]
-            query = (
-                f"SELECT {ids_table['pk_column']}, {column_cast} "
-                f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
-            )
-            with db.engine.begin() as conn:
-                rs = conn.execute(sa.text(query))
-                for row in rs:
-                    record_id = str(row[0])
-                    content = str(row[1])
-
-                    # Find all UUIDs in the content
-                    import re
-
-                    uuid_pattern = re.compile(guid_regexp, re.IGNORECASE)
-                    matches = uuid_pattern.findall(content)
-
-                    for ref_file_id in matches:
+        match ids_table["type"]:
+            case "uuid":
+                # Direct UUID match
+                query = (
+                    f"SELECT {ids_table['pk_column']}, {ids_table['column']} "
+                    f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
+                )
+                with db.engine.begin() as conn:
+                    rs = conn.execute(sa.text(query))
+                    for row in rs:
+                        record_id = str(row[0])
+                        ref_file_id = str(row[1])
                         if ref_file_id not in file_key_map:
                             continue
                         storage_key = file_key_map[ref_file_id]
@@ -1812,6 +1777,50 @@ def file_usage(
                             )
                         total_count += 1
 
+            case "text" | "json":
+                # Extract UUIDs from text/json content
+                column_cast = f"{ids_table['column']}::text" if ids_table["type"] == "json" else ids_table["column"]
+                query = (
+                    f"SELECT {ids_table['pk_column']}, {column_cast} "
+                    f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
+                )
+                with db.engine.begin() as conn:
+                    rs = conn.execute(sa.text(query))
+                    for row in rs:
+                        record_id = str(row[0])
+                        content = str(row[1])
+
+                        # Find all UUIDs in the content
+                        import re
+
+                        uuid_pattern = re.compile(guid_regexp, re.IGNORECASE)
+                        matches = uuid_pattern.findall(content)
+
+                        for ref_file_id in matches:
+                            if ref_file_id not in file_key_map:
+                                continue
+                            storage_key = file_key_map[ref_file_id]
+
+                            # Apply filters
+                            if file_id and ref_file_id != file_id:
+                                continue
+                            if key and not storage_key.endswith(key):
+                                continue
+
+                            # Only collect items within the requested page range
+                            if offset <= total_count < offset + limit:
+                                paginated_usages.append(
+                                    {
+                                        "src": f"{ids_table['table']}.{ids_table['column']}",
+                                        "record_id": record_id,
+                                        "file_id": ref_file_id,
+                                        "key": storage_key,
+                                    }
+                                )
+                            total_count += 1
+            case _:
+                pass
+
     # Output results
     if output_json:
         result = {

api/configs/feature/__init__.py

@@ -1155,6 +1155,16 @@ class CeleryScheduleTasksConfig(BaseSettings):
         default=0,
     )
 
+    # API token last_used_at batch update
+    ENABLE_API_TOKEN_LAST_USED_UPDATE_TASK: bool = Field(
+        description="Enable periodic batch update of API token last_used_at timestamps",
+        default=True,
+    )
+    API_TOKEN_LAST_USED_UPDATE_INTERVAL: int = Field(
+        description="Interval in minutes for batch updating API token last_used_at (default 30)",
+        default=30,
+    )
+
     # Trigger provider refresh (simple version)
     ENABLE_TRIGGER_PROVIDER_REFRESH_TASK: bool = Field(
         description="Enable trigger provider refresh poller",

api/controllers/console/apikey.py

@@ -10,6 +10,7 @@ from libs.helper import TimestampField
 from libs.login import current_account_with_tenant, login_required
 from models.dataset import Dataset
 from models.model import ApiToken, App
+from services.api_token_service import ApiTokenCache
 
 from . import console_ns
 from .wraps import account_initialization_required, edit_permission_required, setup_required
@@ -131,6 +132,11 @@ class BaseApiKeyResource(Resource):
         if key is None:
             flask_restx.abort(HTTPStatus.NOT_FOUND, message="API key not found")
 
+        # Invalidate cache before deleting from database
+        # Type assertion: key is guaranteed to be non-None here because abort() raises
+        assert key is not None  # nosec - for type checker only
+        ApiTokenCache.delete(key.token, key.type)
+
         db.session.query(ApiToken).where(ApiToken.id == api_key_id).delete()
         db.session.commit()
 

api/controllers/console/app/annotation.py

@@ -1,10 +1,11 @@
 from typing import Any, Literal
 
 from flask import abort, make_response, request
-from flask_restx import Resource, fields, marshal, marshal_with
-from pydantic import BaseModel, Field, field_validator
+from flask_restx import Resource
+from pydantic import BaseModel, Field, TypeAdapter, field_validator
 
 from controllers.common.errors import NoFileUploadedError, TooManyFilesError
+from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import (
     account_initialization_required,
@@ -16,9 +17,11 @@ from controllers.console.wraps import (
 )
 from extensions.ext_redis import redis_client
 from fields.annotation_fields import (
-    annotation_fields,
-    annotation_hit_history_fields,
-    build_annotation_model,
+    Annotation,
+    AnnotationExportList,
+    AnnotationHitHistory,
+    AnnotationHitHistoryList,
+    AnnotationList,
 )
 from libs.helper import uuid_value
 from libs.login import login_required
@@ -89,6 +92,14 @@ reg(CreateAnnotationPayload)
 reg(UpdateAnnotationPayload)
 reg(AnnotationReplyStatusQuery)
 reg(AnnotationFilePayload)
+register_schema_models(
+    console_ns,
+    Annotation,
+    AnnotationList,
+    AnnotationExportList,
+    AnnotationHitHistory,
+    AnnotationHitHistoryList,
+)
 
 
 @console_ns.route("/apps/<uuid:app_id>/annotation-reply/<string:action>")
@@ -107,10 +118,11 @@ class AnnotationReplyActionApi(Resource):
     def post(self, app_id, action: Literal["enable", "disable"]):
         app_id = str(app_id)
         args = AnnotationReplyPayload.model_validate(console_ns.payload)
-        if action == "enable":
-            result = AppAnnotationService.enable_app_annotation(args.model_dump(), app_id)
-        elif action == "disable":
-            result = AppAnnotationService.disable_app_annotation(app_id)
+        match action:
+            case "enable":
+                result = AppAnnotationService.enable_app_annotation(args.model_dump(), app_id)
+            case "disable":
+                result = AppAnnotationService.disable_app_annotation(app_id)
         return result, 200
 
 
@@ -201,33 +213,33 @@ class AnnotationApi(Resource):
 
         app_id = str(app_id)
         annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id(app_id, page, limit, keyword)
-        response = {
-            "data": marshal(annotation_list, annotation_fields),
-            "has_more": len(annotation_list) == limit,
-            "limit": limit,
-            "total": total,
-            "page": page,
-        }
-        return response, 200
+        annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
+        response = AnnotationList(
+            data=annotation_models,
+            has_more=len(annotation_list) == limit,
+            limit=limit,
+            total=total,
+            page=page,
+        )
+        return response.model_dump(mode="json"), 200
 
     @console_ns.doc("create_annotation")
     @console_ns.doc(description="Create a new annotation for an app")
     @console_ns.doc(params={"app_id": "Application ID"})
     @console_ns.expect(console_ns.models[CreateAnnotationPayload.__name__])
-    @console_ns.response(201, "Annotation created successfully", build_annotation_model(console_ns))
+    @console_ns.response(201, "Annotation created successfully", console_ns.models[Annotation.__name__])
     @console_ns.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
     @cloud_edition_billing_resource_check("annotation")
-    @marshal_with(annotation_fields)
     @edit_permission_required
     def post(self, app_id):
         app_id = str(app_id)
         args = CreateAnnotationPayload.model_validate(console_ns.payload)
         data = args.model_dump(exclude_none=True)
         annotation = AppAnnotationService.up_insert_app_annotation_from_message(data, app_id)
-        return annotation
+        return Annotation.model_validate(annotation, from_attributes=True).model_dump(mode="json")
 
     @setup_required
     @login_required
@@ -264,7 +276,7 @@ class AnnotationExportApi(Resource):
     @console_ns.response(
         200,
         "Annotations exported successfully",
-        console_ns.model("AnnotationList", {"data": fields.List(fields.Nested(build_annotation_model(console_ns)))}),
+        console_ns.models[AnnotationExportList.__name__],
     )
     @console_ns.response(403, "Insufficient permissions")
     @setup_required
@@ -274,7 +286,8 @@ class AnnotationExportApi(Resource):
     def get(self, app_id):
         app_id = str(app_id)
         annotation_list = AppAnnotationService.export_annotation_list_by_app_id(app_id)
-        response_data = {"data": marshal(annotation_list, annotation_fields)}
+        annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
+        response_data = AnnotationExportList(data=annotation_models).model_dump(mode="json")
 
         # Create response with secure headers for CSV export
         response = make_response(response_data, 200)
@@ -289,7 +302,7 @@ class AnnotationUpdateDeleteApi(Resource):
     @console_ns.doc("update_delete_annotation")
     @console_ns.doc(description="Update or delete an annotation")
     @console_ns.doc(params={"app_id": "Application ID", "annotation_id": "Annotation ID"})
-    @console_ns.response(200, "Annotation updated successfully", build_annotation_model(console_ns))
+    @console_ns.response(200, "Annotation updated successfully", console_ns.models[Annotation.__name__])
     @console_ns.response(204, "Annotation deleted successfully")
     @console_ns.response(403, "Insufficient permissions")
     @console_ns.expect(console_ns.models[UpdateAnnotationPayload.__name__])
@@ -298,7 +311,6 @@ class AnnotationUpdateDeleteApi(Resource):
     @account_initialization_required
     @cloud_edition_billing_resource_check("annotation")
     @edit_permission_required
-    @marshal_with(annotation_fields)
     def post(self, app_id, annotation_id):
         app_id = str(app_id)
         annotation_id = str(annotation_id)
@@ -306,7 +318,7 @@ class AnnotationUpdateDeleteApi(Resource):
         annotation = AppAnnotationService.update_app_annotation_directly(
             args.model_dump(exclude_none=True), app_id, annotation_id
         )
-        return annotation
+        return Annotation.model_validate(annotation, from_attributes=True).model_dump(mode="json")
 
     @setup_required
     @login_required
@@ -414,14 +426,7 @@ class AnnotationHitHistoryListApi(Resource):
     @console_ns.response(
         200,
         "Hit histories retrieved successfully",
-        console_ns.model(
-            "AnnotationHitHistoryList",
-            {
-                "data": fields.List(
-                    fields.Nested(console_ns.model("AnnotationHitHistoryItem", annotation_hit_history_fields))
-                )
-            },
-        ),
+        console_ns.models[AnnotationHitHistoryList.__name__],
     )
     @console_ns.response(403, "Insufficient permissions")
     @setup_required
@@ -436,11 +441,14 @@ class AnnotationHitHistoryListApi(Resource):
         annotation_hit_history_list, total = AppAnnotationService.get_annotation_hit_histories(
             app_id, annotation_id, page, limit
         )
-        response = {
-            "data": marshal(annotation_hit_history_list, annotation_hit_history_fields),
-            "has_more": len(annotation_hit_history_list) == limit,
-            "limit": limit,
-            "total": total,
-            "page": page,
-        }
-        return response
+        history_models = TypeAdapter(list[AnnotationHitHistory]).validate_python(
+            annotation_hit_history_list, from_attributes=True
+        )
+        response = AnnotationHitHistoryList(
+            data=history_models,
+            has_more=len(annotation_hit_history_list) == limit,
+            limit=limit,
+            total=total,
+            page=page,
+        )
+        return response.model_dump(mode="json")

api/controllers/console/app/app.py

@@ -1,3 +1,4 @@
+import logging
 import uuid
 from datetime import datetime
 from typing import Any, Literal, TypeAlias
@@ -54,6 +55,8 @@ ALLOW_CREATE_APP_MODES = ["chat", "agent-chat", "advanced-chat", "workflow", "co
 
 register_enum_models(console_ns, IconType)
 
+_logger = logging.getLogger(__name__)
+
 
 class AppListQuery(BaseModel):
     page: int = Field(default=1, ge=1, le=99999, description="Page number (1-99999)")
@@ -499,6 +502,7 @@ class AppListApi(Resource):
                     select(Workflow).where(
                         Workflow.version == Workflow.VERSION_DRAFT,
                         Workflow.app_id.in_(workflow_capable_app_ids),
+                        Workflow.tenant_id == current_tenant_id,
                     )
                 )
                 .scalars()
@@ -510,12 +514,14 @@ class AppListApi(Resource):
                 NodeType.TRIGGER_PLUGIN,
             }
             for workflow in draft_workflows:
+                node_id = None
                 try:
-                    for _, node_data in workflow.walk_nodes():
+                    for node_id, node_data in workflow.walk_nodes():
                         if node_data.get("type") in trigger_node_types:
                             draft_trigger_app_ids.add(str(workflow.app_id))
                             break
                 except Exception:
+                    _logger.exception("error while walking nodes, workflow_id=%s, node_id=%s", workflow.id, node_id)
                     continue
 
         for app in app_pagination.items:

api/controllers/console/app/audio.py

@@ -6,6 +6,7 @@ from pydantic import BaseModel, Field
 from werkzeug.exceptions import InternalServerError
 
 import services
+from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.app.error import (
     AppUnavailableError,
@@ -33,7 +34,6 @@ from services.errors.audio import (
 )
 
 logger = logging.getLogger(__name__)
-DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
 
 
 class TextToSpeechPayload(BaseModel):
@@ -47,13 +47,11 @@ class TextToSpeechVoiceQuery(BaseModel):
     language: str = Field(..., description="Language code")
 
 
-console_ns.schema_model(
-    TextToSpeechPayload.__name__, TextToSpeechPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)
-)
-console_ns.schema_model(
-    TextToSpeechVoiceQuery.__name__,
-    TextToSpeechVoiceQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0),
-)
+class AudioTranscriptResponse(BaseModel):
+    text: str = Field(description="Transcribed text from audio")
+
+
+register_schema_models(console_ns, AudioTranscriptResponse, TextToSpeechPayload, TextToSpeechVoiceQuery)
 
 
 @console_ns.route("/apps/<uuid:app_id>/audio-to-text")
@@ -64,7 +62,7 @@ class ChatMessageAudioApi(Resource):
     @console_ns.response(
         200,
         "Audio transcription successful",
-        console_ns.model("AudioTranscriptResponse", {"text": fields.String(description="Transcribed text from audio")}),
+        console_ns.models[AudioTranscriptResponse.__name__],
     )
     @console_ns.response(400, "Bad request - No audio uploaded or unsupported type")
     @console_ns.response(413, "Audio file too large")

api/controllers/console/app/conversation.py

@@ -508,16 +508,19 @@ class ChatConversationApi(Resource):
                 case "created_at" | "-created_at" | _:
                     query = query.where(Conversation.created_at <= end_datetime_utc)
 
-        if args.annotation_status == "annotated":
-            query = query.options(joinedload(Conversation.message_annotations)).join(  # type: ignore
-                MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
-            )
-        elif args.annotation_status == "not_annotated":
-            query = (
-                query.outerjoin(MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id)
-                .group_by(Conversation.id)
-                .having(func.count(MessageAnnotation.id) == 0)
-            )
+        match args.annotation_status:
+            case "annotated":
+                query = query.options(joinedload(Conversation.message_annotations)).join(  # type: ignore
+                    MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
+                )
+            case "not_annotated":
+                query = (
+                    query.outerjoin(MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id)
+                    .group_by(Conversation.id)
+                    .having(func.count(MessageAnnotation.id) == 0)
+                )
+            case "all":
+                pass
 
         if app_model.mode == AppMode.ADVANCED_CHAT:
             query = query.where(Conversation.invoke_from != InvokeFrom.DEBUGGER)

api/controllers/console/app/message.py

@@ -7,6 +7,7 @@ from pydantic import BaseModel, Field, field_validator
 from sqlalchemy import exists, select
 from werkzeug.exceptions import InternalServerError, NotFound
 
+from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.app.error import (
     CompletionRequestError,
@@ -35,7 +36,6 @@ from services.errors.message import MessageNotExistsError, SuggestedQuestionsAft
 from services.message_service import MessageService
 
 logger = logging.getLogger(__name__)
-DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
 
 
 class ChatMessagesQuery(BaseModel):
@@ -90,13 +90,22 @@ class FeedbackExportQuery(BaseModel):
         raise ValueError("has_comment must be a boolean value")
 
 
-def reg(cls: type[BaseModel]):
-    console_ns.schema_model(cls.__name__, cls.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0))
+class AnnotationCountResponse(BaseModel):
+    count: int = Field(description="Number of annotations")
 
 
-reg(ChatMessagesQuery)
-reg(MessageFeedbackPayload)
-reg(FeedbackExportQuery)
+class SuggestedQuestionsResponse(BaseModel):
+    data: list[str] = Field(description="Suggested question")
+
+
+register_schema_models(
+    console_ns,
+    ChatMessagesQuery,
+    MessageFeedbackPayload,
+    FeedbackExportQuery,
+    AnnotationCountResponse,
+    SuggestedQuestionsResponse,
+)
 
 # Register models for flask_restx to avoid dict type issues in Swagger
 # Register in dependency order: base models first, then dependent models
@@ -231,7 +240,7 @@ class ChatMessageListApi(Resource):
     @marshal_with(message_infinite_scroll_pagination_model)
     @edit_permission_required
     def get(self, app_model):
-        args = ChatMessagesQuery.model_validate(request.args.to_dict(flat=True))  # type: ignore
+        args = ChatMessagesQuery.model_validate(request.args.to_dict())
 
         conversation = (
             db.session.query(Conversation)
@@ -356,7 +365,7 @@ class MessageAnnotationCountApi(Resource):
     @console_ns.response(
         200,
         "Annotation count retrieved successfully",
-        console_ns.model("AnnotationCountResponse", {"count": fields.Integer(description="Number of annotations")}),
+        console_ns.models[AnnotationCountResponse.__name__],
     )
     @get_app_model
     @setup_required
@@ -376,9 +385,7 @@ class MessageSuggestedQuestionApi(Resource):
     @console_ns.response(
         200,
         "Suggested questions retrieved successfully",
-        console_ns.model(
-            "SuggestedQuestionsResponse", {"data": fields.List(fields.String(description="Suggested question"))}
-        ),
+        console_ns.models[SuggestedQuestionsResponse.__name__],
     )
     @console_ns.response(404, "Message or conversation not found")
     @setup_required
@@ -428,7 +435,7 @@ class MessageFeedbackExportApi(Resource):
     @login_required
     @account_initialization_required
     def get(self, app_model):
-        args = FeedbackExportQuery.model_validate(request.args.to_dict(flat=True))  # type: ignore
+        args = FeedbackExportQuery.model_validate(request.args.to_dict())
 
         # Import the service function
         from services.feedback_service import FeedbackService

api/controllers/console/auth/data_source_oauth.py

@@ -2,9 +2,11 @@ import logging
 
 import httpx
 from flask import current_app, redirect, request
-from flask_restx import Resource, fields
+from flask_restx import Resource
+from pydantic import BaseModel, Field
 
 from configs import dify_config
+from controllers.common.schema import register_schema_models
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
 
@@ -14,6 +16,26 @@ from ..wraps import account_initialization_required, is_admin_or_owner_required,
 logger = logging.getLogger(__name__)
 
 
+class OAuthDataSourceResponse(BaseModel):
+    data: str = Field(description="Authorization URL or 'internal' for internal setup")
+
+
+class OAuthDataSourceBindingResponse(BaseModel):
+    result: str = Field(description="Operation result")
+
+
+class OAuthDataSourceSyncResponse(BaseModel):
+    result: str = Field(description="Operation result")
+
+
+register_schema_models(
+    console_ns,
+    OAuthDataSourceResponse,
+    OAuthDataSourceBindingResponse,
+    OAuthDataSourceSyncResponse,
+)
+
+
 def get_oauth_providers():
     with current_app.app_context():
         notion_oauth = NotionOAuth(
@@ -34,10 +56,7 @@ class OAuthDataSource(Resource):
     @console_ns.response(
         200,
         "Authorization URL or internal setup success",
-        console_ns.model(
-            "OAuthDataSourceResponse",
-            {"data": fields.Raw(description="Authorization URL or 'internal' for internal setup")},
-        ),
+        console_ns.models[OAuthDataSourceResponse.__name__],
     )
     @console_ns.response(400, "Invalid provider")
     @console_ns.response(403, "Admin privileges required")
@@ -101,7 +120,7 @@ class OAuthDataSourceBinding(Resource):
     @console_ns.response(
         200,
         "Data source binding success",
-        console_ns.model("OAuthDataSourceBindingResponse", {"result": fields.String(description="Operation result")}),
+        console_ns.models[OAuthDataSourceBindingResponse.__name__],
     )
     @console_ns.response(400, "Invalid provider or code")
     def get(self, provider: str):
@@ -133,7 +152,7 @@ class OAuthDataSourceSync(Resource):
     @console_ns.response(
         200,
         "Data source sync success",
-        console_ns.model("OAuthDataSourceSyncResponse", {"result": fields.String(description="Operation result")}),
+        console_ns.models[OAuthDataSourceSyncResponse.__name__],
     )
     @console_ns.response(400, "Invalid provider or sync failed")
     @setup_required

api/controllers/console/auth/forgot_password.py

@@ -2,10 +2,11 @@ import base64
 import secrets
 
 from flask import request
-from flask_restx import Resource, fields
+from flask_restx import Resource
 from pydantic import BaseModel, Field, field_validator
 from sqlalchemy.orm import Session
 
+from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.auth.error import (
     EmailCodeError,
@@ -48,8 +49,31 @@ class ForgotPasswordResetPayload(BaseModel):
         return valid_password(value)
 
 
-for model in (ForgotPasswordSendPayload, ForgotPasswordCheckPayload, ForgotPasswordResetPayload):
-    console_ns.schema_model(model.__name__, model.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0))
+class ForgotPasswordEmailResponse(BaseModel):
+    result: str = Field(description="Operation result")
+    data: str | None = Field(default=None, description="Reset token")
+    code: str | None = Field(default=None, description="Error code if account not found")
+
+
+class ForgotPasswordCheckResponse(BaseModel):
+    is_valid: bool = Field(description="Whether code is valid")
+    email: EmailStr = Field(description="Email address")
+    token: str = Field(description="New reset token")
+
+
+class ForgotPasswordResetResponse(BaseModel):
+    result: str = Field(description="Operation result")
+
+
+register_schema_models(
+    console_ns,
+    ForgotPasswordSendPayload,
+    ForgotPasswordCheckPayload,
+    ForgotPasswordResetPayload,
+    ForgotPasswordEmailResponse,
+    ForgotPasswordCheckResponse,
+    ForgotPasswordResetResponse,
+)
 
 
 @console_ns.route("/forgot-password")
@@ -60,14 +84,7 @@ class ForgotPasswordSendEmailApi(Resource):
     @console_ns.response(
         200,
         "Email sent successfully",
-        console_ns.model(
-            "ForgotPasswordEmailResponse",
-            {
-                "result": fields.String(description="Operation result"),
-                "data": fields.String(description="Reset token"),
-                "code": fields.String(description="Error code if account not found"),
-            },
-        ),
+        console_ns.models[ForgotPasswordEmailResponse.__name__],
     )
     @console_ns.response(400, "Invalid email or rate limit exceeded")
     @setup_required
@@ -106,14 +123,7 @@ class ForgotPasswordCheckApi(Resource):
     @console_ns.response(
         200,
         "Code verified successfully",
-        console_ns.model(
-            "ForgotPasswordCheckResponse",
-            {
-                "is_valid": fields.Boolean(description="Whether code is valid"),
-                "email": fields.String(description="Email address"),
-                "token": fields.String(description="New reset token"),
-            },
-        ),
+        console_ns.models[ForgotPasswordCheckResponse.__name__],
     )
     @console_ns.response(400, "Invalid code or token")
     @setup_required
@@ -163,7 +173,7 @@ class ForgotPasswordResetApi(Resource):
     @console_ns.response(
         200,
         "Password reset successfully",
-        console_ns.model("ForgotPasswordResetResponse", {"result": fields.String(description="Operation result")}),
+        console_ns.models[ForgotPasswordResetResponse.__name__],
     )
     @console_ns.response(400, "Invalid token or password mismatch")
     @setup_required

api/controllers/console/auth/oauth_server.py

@@ -155,43 +155,43 @@ class OAuthServerUserTokenApi(Resource):
             grant_type = OAuthGrantType(payload.grant_type)
         except ValueError:
             raise BadRequest("invalid grant_type")
+        match grant_type:
+            case OAuthGrantType.AUTHORIZATION_CODE:
+                if not payload.code:
+                    raise BadRequest("code is required")
 
-        if grant_type == OAuthGrantType.AUTHORIZATION_CODE:
-            if not payload.code:
-                raise BadRequest("code is required")
+                if payload.client_secret != oauth_provider_app.client_secret:
+                    raise BadRequest("client_secret is invalid")
 
-            if payload.client_secret != oauth_provider_app.client_secret:
-                raise BadRequest("client_secret is invalid")
+                if payload.redirect_uri not in oauth_provider_app.redirect_uris:
+                    raise BadRequest("redirect_uri is invalid")
 
-            if payload.redirect_uri not in oauth_provider_app.redirect_uris:
-                raise BadRequest("redirect_uri is invalid")
+                access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
+                    grant_type, code=payload.code, client_id=oauth_provider_app.client_id
+                )
+                return jsonable_encoder(
+                    {
+                        "access_token": access_token,
+                        "token_type": "Bearer",
+                        "expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
+                        "refresh_token": refresh_token,
+                    }
+                )
+            case OAuthGrantType.REFRESH_TOKEN:
+                if not payload.refresh_token:
+                    raise BadRequest("refresh_token is required")
 
-            access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
-                grant_type, code=payload.code, client_id=oauth_provider_app.client_id
-            )
-            return jsonable_encoder(
-                {
-                    "access_token": access_token,
-                    "token_type": "Bearer",
-                    "expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
-                    "refresh_token": refresh_token,
-                }
-            )
-        elif grant_type == OAuthGrantType.REFRESH_TOKEN:
-            if not payload.refresh_token:
-                raise BadRequest("refresh_token is required")
-
-            access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
-                grant_type, refresh_token=payload.refresh_token, client_id=oauth_provider_app.client_id
-            )
-            return jsonable_encoder(
-                {
-                    "access_token": access_token,
-                    "token_type": "Bearer",
-                    "expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
-                    "refresh_token": refresh_token,
-                }
-            )
+                access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
+                    grant_type, refresh_token=payload.refresh_token, client_id=oauth_provider_app.client_id
+                )
+                return jsonable_encoder(
+                    {
+                        "access_token": access_token,
+                        "token_type": "Bearer",
+                        "expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
+                        "refresh_token": refresh_token,
+                    }
+                )
 
 
 @console_ns.route("/oauth/provider/account")

api/controllers/console/datasets/data_source.py

@@ -1,6 +1,6 @@
 import json
 from collections.abc import Generator
-from typing import Any, cast
+from typing import Any, Literal, cast
 
 from flask import request
 from flask_restx import Resource, fields, marshal_with
@@ -157,9 +157,8 @@ class DataSourceApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def patch(self, binding_id, action):
+    def patch(self, binding_id, action: Literal["enable", "disable"]):
         binding_id = str(binding_id)
-        action = str(action)
         with Session(db.engine) as session:
             data_source_binding = session.execute(
                 select(DataSourceOauthBinding).filter_by(id=binding_id)
@@ -167,23 +166,24 @@ class DataSourceApi(Resource):
         if data_source_binding is None:
             raise NotFound("Data source binding not found.")
         # enable binding
-        if action == "enable":
-            if data_source_binding.disabled:
-                data_source_binding.disabled = False
-                data_source_binding.updated_at = naive_utc_now()
-                db.session.add(data_source_binding)
-                db.session.commit()
-            else:
-                raise ValueError("Data source is not disabled.")
-        # disable binding
-        if action == "disable":
-            if not data_source_binding.disabled:
-                data_source_binding.disabled = True
-                data_source_binding.updated_at = naive_utc_now()
-                db.session.add(data_source_binding)
-                db.session.commit()
-            else:
-                raise ValueError("Data source is disabled.")
+        match action:
+            case "enable":
+                if data_source_binding.disabled:
+                    data_source_binding.disabled = False
+                    data_source_binding.updated_at = naive_utc_now()
+                    db.session.add(data_source_binding)
+                    db.session.commit()
+                else:
+                    raise ValueError("Data source is not disabled.")
+            # disable binding
+            case "disable":
+                if not data_source_binding.disabled:
+                    data_source_binding.disabled = True
+                    data_source_binding.updated_at = naive_utc_now()
+                    db.session.add(data_source_binding)
+                    db.session.commit()
+                else:
+                    raise ValueError("Data source is disabled.")
         return {"result": "success"}, 200
 
 

api/controllers/console/datasets/datasets.py

@@ -55,6 +55,7 @@ from libs.login import current_account_with_tenant, login_required
 from models import ApiToken, Dataset, Document, DocumentSegment, UploadFile
 from models.dataset import DatasetPermissionEnum
 from models.provider_ids import ModelProviderID
+from services.api_token_service import ApiTokenCache
 from services.dataset_service import DatasetPermissionService, DatasetService, DocumentService
 
 # Register models for flask_restx to avoid dict type issues in Swagger
@@ -820,6 +821,11 @@ class DatasetApiDeleteApi(Resource):
         if key is None:
             console_ns.abort(404, message="API key not found")
 
+        # Invalidate cache before deleting from database
+        # Type assertion: key is guaranteed to be non-None here because abort() raises
+        assert key is not None  # nosec - for type checker only
+        ApiTokenCache.delete(key.token, key.type)
+
         db.session.query(ApiToken).where(ApiToken.id == api_key_id).delete()
         db.session.commit()
 

api/controllers/console/datasets/datasets_document.py

@@ -576,63 +576,62 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
             if document.indexing_status in {"completed", "error"}:
                 raise DocumentAlreadyFinishedError()
             data_source_info = document.data_source_info_dict
+            match document.data_source_type:
+                case "upload_file":
+                    if not data_source_info:
+                        continue
+                    file_id = data_source_info["upload_file_id"]
+                    file_detail = (
+                        db.session.query(UploadFile)
+                        .where(UploadFile.tenant_id == current_tenant_id, UploadFile.id == file_id)
+                        .first()
+                    )
 
-            if document.data_source_type == "upload_file":
-                if not data_source_info:
-                    continue
-                file_id = data_source_info["upload_file_id"]
-                file_detail = (
-                    db.session.query(UploadFile)
-                    .where(UploadFile.tenant_id == current_tenant_id, UploadFile.id == file_id)
-                    .first()
-                )
+                    if file_detail is None:
+                        raise NotFound("File not found.")
 
-                if file_detail is None:
-                    raise NotFound("File not found.")
+                    extract_setting = ExtractSetting(
+                        datasource_type=DatasourceType.FILE, upload_file=file_detail, document_model=document.doc_form
+                    )
+                    extract_settings.append(extract_setting)
+                case "notion_import":
+                    if not data_source_info:
+                        continue
+                    extract_setting = ExtractSetting(
+                        datasource_type=DatasourceType.NOTION,
+                        notion_info=NotionInfo.model_validate(
+                            {
+                                "credential_id": data_source_info.get("credential_id"),
+                                "notion_workspace_id": data_source_info["notion_workspace_id"],
+                                "notion_obj_id": data_source_info["notion_page_id"],
+                                "notion_page_type": data_source_info["type"],
+                                "tenant_id": current_tenant_id,
+                            }
+                        ),
+                        document_model=document.doc_form,
+                    )
+                    extract_settings.append(extract_setting)
+                case "website_crawl":
+                    if not data_source_info:
+                        continue
+                    extract_setting = ExtractSetting(
+                        datasource_type=DatasourceType.WEBSITE,
+                        website_info=WebsiteInfo.model_validate(
+                            {
+                                "provider": data_source_info["provider"],
+                                "job_id": data_source_info["job_id"],
+                                "url": data_source_info["url"],
+                                "tenant_id": current_tenant_id,
+                                "mode": data_source_info["mode"],
+                                "only_main_content": data_source_info["only_main_content"],
+                            }
+                        ),
+                        document_model=document.doc_form,
+                    )
+                    extract_settings.append(extract_setting)
 
-                extract_setting = ExtractSetting(
-                    datasource_type=DatasourceType.FILE, upload_file=file_detail, document_model=document.doc_form
-                )
-                extract_settings.append(extract_setting)
-
-            elif document.data_source_type == "notion_import":
-                if not data_source_info:
-                    continue
-                extract_setting = ExtractSetting(
-                    datasource_type=DatasourceType.NOTION,
-                    notion_info=NotionInfo.model_validate(
-                        {
-                            "credential_id": data_source_info.get("credential_id"),
-                            "notion_workspace_id": data_source_info["notion_workspace_id"],
-                            "notion_obj_id": data_source_info["notion_page_id"],
-                            "notion_page_type": data_source_info["type"],
-                            "tenant_id": current_tenant_id,
-                        }
-                    ),
-                    document_model=document.doc_form,
-                )
-                extract_settings.append(extract_setting)
-            elif document.data_source_type == "website_crawl":
-                if not data_source_info:
-                    continue
-                extract_setting = ExtractSetting(
-                    datasource_type=DatasourceType.WEBSITE,
-                    website_info=WebsiteInfo.model_validate(
-                        {
-                            "provider": data_source_info["provider"],
-                            "job_id": data_source_info["job_id"],
-                            "url": data_source_info["url"],
-                            "tenant_id": current_tenant_id,
-                            "mode": data_source_info["mode"],
-                            "only_main_content": data_source_info["only_main_content"],
-                        }
-                    ),
-                    document_model=document.doc_form,
-                )
-                extract_settings.append(extract_setting)
-
-            else:
-                raise ValueError("Data source type not support")
+                case _:
+                    raise ValueError("Data source type not support")
             indexing_runner = IndexingRunner()
             try:
                 response = indexing_runner.indexing_estimate(
@@ -954,23 +953,24 @@ class DocumentProcessingApi(DocumentResource):
         if not current_user.is_dataset_editor:
             raise Forbidden()
 
-        if action == "pause":
-            if document.indexing_status != "indexing":
-                raise InvalidActionError("Document not in indexing state.")
+        match action:
+            case "pause":
+                if document.indexing_status != "indexing":
+                    raise InvalidActionError("Document not in indexing state.")
 
-            document.paused_by = current_user.id
-            document.paused_at = naive_utc_now()
-            document.is_paused = True
-            db.session.commit()
+                document.paused_by = current_user.id
+                document.paused_at = naive_utc_now()
+                document.is_paused = True
+                db.session.commit()
 
-        elif action == "resume":
-            if document.indexing_status not in {"paused", "error"}:
-                raise InvalidActionError("Document not in paused or error state.")
+            case "resume":
+                if document.indexing_status not in {"paused", "error"}:
+                    raise InvalidActionError("Document not in paused or error state.")
 
-            document.paused_by = None
-            document.paused_at = None
-            document.is_paused = False
-            db.session.commit()
+                document.paused_by = None
+                document.paused_at = None
+                document.is_paused = False
+                db.session.commit()
 
         return {"result": "success"}, 200
 
@@ -1339,6 +1339,18 @@ class DocumentGenerateSummaryApi(Resource):
             missing_ids = set(document_list) - found_ids
             raise NotFound(f"Some documents not found: {list(missing_ids)}")
 
+        # Update need_summary to True for documents that don't have it set
+        # This handles the case where documents were created when summary_index_setting was disabled
+        documents_to_update = [doc for doc in documents if not doc.need_summary and doc.doc_form != "qa_model"]
+
+        if documents_to_update:
+            document_ids_to_update = [str(doc.id) for doc in documents_to_update]
+            DocumentService.update_documents_need_summary(
+                dataset_id=dataset_id,
+                document_ids=document_ids_to_update,
+                need_summary=True,
+            )
+
         # Dispatch async tasks for each document
         for document in documents:
             # Skip qa_model documents as they don't generate summaries

api/controllers/console/datasets/metadata.py

@@ -126,10 +126,11 @@ class DatasetMetadataBuiltInFieldActionApi(Resource):
             raise NotFound("Dataset not found.")
         DatasetService.check_dataset_permission(dataset, current_user)
 
-        if action == "enable":
-            MetadataService.enable_built_in_field(dataset)
-        elif action == "disable":
-            MetadataService.disable_built_in_field(dataset)
+        match action:
+            case "enable":
+                MetadataService.enable_built_in_field(dataset)
+            case "disable":
+                MetadataService.disable_built_in_field(dataset)
         return {"result": "success"}, 200
 
 

api/controllers/console/explore/trial.py

@@ -1,8 +1,9 @@
 import logging
-from typing import Any, cast
+from typing import Any, Literal, cast
 
 from flask import request
-from flask_restx import Resource, fields, marshal, marshal_with, reqparse
+from flask_restx import Resource, fields, marshal, marshal_with
+from pydantic import BaseModel
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
 import services
@@ -51,7 +52,7 @@ from fields.app_fields import (
     tag_fields,
 )
 from fields.dataset_fields import dataset_fields
-from fields.member_fields import build_simple_account_model
+from fields.member_fields import simple_account_fields
 from fields.workflow_fields import (
     conversation_variable_fields,
     pipeline_variable_fields,
@@ -103,7 +104,7 @@ app_detail_fields_with_site_copy["tags"] = fields.List(fields.Nested(tag_model))
 app_detail_fields_with_site_copy["site"] = fields.Nested(site_model)
 app_detail_with_site_model = get_or_create_model("TrialAppDetailWithSite", app_detail_fields_with_site_copy)
 
-simple_account_model = build_simple_account_model(console_ns)
+simple_account_model = get_or_create_model("SimpleAccount", simple_account_fields)
 conversation_variable_model = get_or_create_model("TrialConversationVariable", conversation_variable_fields)
 pipeline_variable_model = get_or_create_model("TrialPipelineVariable", pipeline_variable_fields)
 
@@ -117,7 +118,56 @@ workflow_fields_copy["rag_pipeline_variables"] = fields.List(fields.Nested(pipel
 workflow_model = get_or_create_model("TrialWorkflow", workflow_fields_copy)
 
 
+# Pydantic models for request validation
+DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
+
+
+class WorkflowRunRequest(BaseModel):
+    inputs: dict
+    files: list | None = None
+
+
+class ChatRequest(BaseModel):
+    inputs: dict
+    query: str
+    files: list | None = None
+    conversation_id: str | None = None
+    parent_message_id: str | None = None
+    retriever_from: str = "explore_app"
+
+
+class TextToSpeechRequest(BaseModel):
+    message_id: str | None = None
+    voice: str | None = None
+    text: str | None = None
+    streaming: bool | None = None
+
+
+class CompletionRequest(BaseModel):
+    inputs: dict
+    query: str = ""
+    files: list | None = None
+    response_mode: Literal["blocking", "streaming"] | None = None
+    retriever_from: str = "explore_app"
+
+
+# Register schemas for Swagger documentation
+console_ns.schema_model(
+    WorkflowRunRequest.__name__, WorkflowRunRequest.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)
+)
+console_ns.schema_model(
+    ChatRequest.__name__, ChatRequest.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)
+)
+console_ns.schema_model(
+    TextToSpeechRequest.__name__, TextToSpeechRequest.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)
+)
+console_ns.schema_model(
+    CompletionRequest.__name__, CompletionRequest.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)
+)
+
+
 class TrialAppWorkflowRunApi(TrialAppResource):
+    @console_ns.expect(console_ns.models[WorkflowRunRequest.__name__])
     def post(self, trial_app):
         """
         Run workflow
@@ -129,10 +179,8 @@ class TrialAppWorkflowRunApi(TrialAppResource):
         if app_mode != AppMode.WORKFLOW:
             raise NotWorkflowAppError()
 
-        parser = reqparse.RequestParser()
-        parser.add_argument("inputs", type=dict, required=True, nullable=False, location="json")
-        parser.add_argument("files", type=list, required=False, location="json")
-        args = parser.parse_args()
+        request_data = WorkflowRunRequest.model_validate(console_ns.payload)
+        args = request_data.model_dump()
         assert current_user is not None
         try:
             app_id = app_model.id
@@ -183,6 +231,7 @@ class TrialAppWorkflowTaskStopApi(TrialAppResource):
 
 
 class TrialChatApi(TrialAppResource):
+    @console_ns.expect(console_ns.models[ChatRequest.__name__])
     @trial_feature_enable
     def post(self, trial_app):
         app_model = trial_app
@@ -190,14 +239,14 @@ class TrialChatApi(TrialAppResource):
         if app_mode not in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.ADVANCED_CHAT}:
             raise NotChatAppError()
 
-        parser = reqparse.RequestParser()
-        parser.add_argument("inputs", type=dict, required=True, location="json")
-        parser.add_argument("query", type=str, required=True, location="json")
-        parser.add_argument("files", type=list, required=False, location="json")
-        parser.add_argument("conversation_id", type=uuid_value, location="json")
-        parser.add_argument("parent_message_id", type=uuid_value, required=False, location="json")
-        parser.add_argument("retriever_from", type=str, required=False, default="explore_app", location="json")
-        args = parser.parse_args()
+        request_data = ChatRequest.model_validate(console_ns.payload)
+        args = request_data.model_dump()
+
+        # Validate UUID values if provided
+        if args.get("conversation_id"):
+            args["conversation_id"] = uuid_value(args["conversation_id"])
+        if args.get("parent_message_id"):
+            args["parent_message_id"] = uuid_value(args["parent_message_id"])
 
         args["auto_generate_name"] = False
 
@@ -320,20 +369,16 @@ class TrialChatAudioApi(TrialAppResource):
 
 
 class TrialChatTextApi(TrialAppResource):
+    @console_ns.expect(console_ns.models[TextToSpeechRequest.__name__])
     @trial_feature_enable
     def post(self, trial_app):
         app_model = trial_app
         try:
-            parser = reqparse.RequestParser()
-            parser.add_argument("message_id", type=str, required=False, location="json")
-            parser.add_argument("voice", type=str, location="json")
-            parser.add_argument("text", type=str, location="json")
-            parser.add_argument("streaming", type=bool, location="json")
-            args = parser.parse_args()
+            request_data = TextToSpeechRequest.model_validate(console_ns.payload)
 
-            message_id = args.get("message_id", None)
-            text = args.get("text", None)
-            voice = args.get("voice", None)
+            message_id = request_data.message_id
+            text = request_data.text
+            voice = request_data.voice
             if not isinstance(current_user, Account):
                 raise ValueError("current_user must be an Account instance")
 
@@ -371,19 +416,15 @@ class TrialChatTextApi(TrialAppResource):
 
 
 class TrialCompletionApi(TrialAppResource):
+    @console_ns.expect(console_ns.models[CompletionRequest.__name__])
     @trial_feature_enable
     def post(self, trial_app):
         app_model = trial_app
         if app_model.mode != "completion":
             raise NotCompletionAppError()
 
-        parser = reqparse.RequestParser()
-        parser.add_argument("inputs", type=dict, required=True, location="json")
-        parser.add_argument("query", type=str, location="json", default="")
-        parser.add_argument("files", type=list, required=False, location="json")
-        parser.add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json")
-        parser.add_argument("retriever_from", type=str, required=False, default="explore_app", location="json")
-        args = parser.parse_args()
+        request_data = CompletionRequest.model_validate(console_ns.payload)
+        args = request_data.model_dump()
 
         streaming = args["response_mode"] == "streaming"
         args["auto_generate_name"] = False

api/controllers/console/feature.py

@@ -1,58 +1,60 @@
-from pydantic import BaseModel, Field
+from flask_restx import Resource, fields
 from werkzeug.exceptions import Unauthorized
 
-from controllers.fastopenapi import console_router
 from libs.login import current_account_with_tenant, current_user, login_required
-from services.feature_service import FeatureModel, FeatureService, SystemFeatureModel
+from services.feature_service import FeatureService
 
+from . import console_ns
 from .wraps import account_initialization_required, cloud_utm_record, setup_required
 
 
-class FeatureResponse(BaseModel):
-    features: FeatureModel = Field(description="Feature configuration object")
+@console_ns.route("/features")
+class FeatureApi(Resource):
+    @console_ns.doc("get_tenant_features")
+    @console_ns.doc(description="Get feature configuration for current tenant")
+    @console_ns.response(
+        200,
+        "Success",
+        console_ns.model("FeatureResponse", {"features": fields.Raw(description="Feature configuration object")}),
+    )
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_utm_record
+    def get(self):
+        """Get feature configuration for current tenant"""
+        _, current_tenant_id = current_account_with_tenant()
+
+        return FeatureService.get_features(current_tenant_id).model_dump()
 
 
-class SystemFeatureResponse(BaseModel):
-    features: SystemFeatureModel = Field(description="System feature configuration object")
+@console_ns.route("/system-features")
+class SystemFeatureApi(Resource):
+    @console_ns.doc("get_system_features")
+    @console_ns.doc(description="Get system-wide feature configuration")
+    @console_ns.response(
+        200,
+        "Success",
+        console_ns.model(
+            "SystemFeatureResponse", {"features": fields.Raw(description="System feature configuration object")}
+        ),
+    )
+    def get(self):
+        """Get system-wide feature configuration
 
+        NOTE: This endpoint is unauthenticated by design, as it provides system features
+        data required for dashboard initialization.
 
-@console_router.get(
-    "/features",
-    response_model=FeatureResponse,
-    tags=["console"],
-)
-@setup_required
-@login_required
-@account_initialization_required
-@cloud_utm_record
-def get_tenant_features() -> FeatureResponse:
-    """Get feature configuration for current tenant."""
-    _, current_tenant_id = current_account_with_tenant()
+        Authentication would create circular dependency (can't login without dashboard loading).
 
-    return FeatureResponse(features=FeatureService.get_features(current_tenant_id))
-
-
-@console_router.get(
-    "/system-features",
-    response_model=SystemFeatureResponse,
-    tags=["console"],
-)
-def get_system_features() -> SystemFeatureResponse:
-    """Get system-wide feature configuration
-
-    NOTE: This endpoint is unauthenticated by design, as it provides system features
-    data required for dashboard initialization.
-
-    Authentication would create circular dependency (can't login without dashboard loading).
-
-    Only non-sensitive configuration data should be returned by this endpoint.
-    """
-    # NOTE(QuantumGhost): ideally we should access `current_user.is_authenticated`
-    # without a try-catch. However, due to the implementation of user loader (the `load_user_from_request`
-    # in api/extensions/ext_login.py), accessing `current_user.is_authenticated` will
-    # raise `Unauthorized` exception if authentication token is not provided.
-    try:
-        is_authenticated = current_user.is_authenticated
-    except Unauthorized:
-        is_authenticated = False
-    return SystemFeatureResponse(features=FeatureService.get_system_features(is_authenticated=is_authenticated))
+        Only non-sensitive configuration data should be returned by this endpoint.
+        """
+        # NOTE(QuantumGhost): ideally we should access `current_user.is_authenticated`
+        # without a try-catch. However, due to the implementation of user loader (the `load_user_from_request`
+        # in api/extensions/ext_login.py), accessing `current_user.is_authenticated` will
+        # raise `Unauthorized` exception if authentication token is not provided.
+        try:
+            is_authenticated = current_user.is_authenticated
+        except Unauthorized:
+            is_authenticated = False
+        return FeatureService.get_system_features(is_authenticated=is_authenticated).model_dump()

api/controllers/console/tag/tags.py

@@ -1,14 +1,27 @@
 from typing import Literal
-from uuid import UUID
 
+from flask import request
+from flask_restx import Namespace, Resource, fields, marshal_with
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import Forbidden
 
+from controllers.common.schema import register_schema_models
+from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required
-from controllers.fastopenapi import console_router
 from libs.login import current_account_with_tenant, login_required
 from services.tag_service import TagService
 
+dataset_tag_fields = {
+    "id": fields.String,
+    "name": fields.String,
+    "type": fields.String,
+    "binding_count": fields.String,
+}
+
+
+def build_dataset_tag_fields(api_or_ns: Namespace):
+    return api_or_ns.model("DataSetTag", dataset_tag_fields)
+
 
 class TagBasePayload(BaseModel):
     name: str = Field(description="Tag name", min_length=1, max_length=50)
@@ -32,129 +45,115 @@ class TagListQueryParam(BaseModel):
     keyword: str | None = Field(None, description="Search keyword")
 
 
-class TagResponse(BaseModel):
-    id: str = Field(description="Tag ID")
-    name: str = Field(description="Tag name")
-    type: str = Field(description="Tag type")
-    binding_count: int = Field(description="Number of bindings")
-
-
-class TagBindingResult(BaseModel):
-    result: Literal["success"] = Field(description="Operation result", examples=["success"])
-
-
-@console_router.get(
-    "/tags",
-    response_model=list[TagResponse],
-    tags=["console"],
+register_schema_models(
+    console_ns,
+    TagBasePayload,
+    TagBindingPayload,
+    TagBindingRemovePayload,
+    TagListQueryParam,
 )
-@setup_required
-@login_required
-@account_initialization_required
-def list_tags(query: TagListQueryParam) -> list[TagResponse]:
-    _, current_tenant_id = current_account_with_tenant()
-    tags = TagService.get_tags(query.type, current_tenant_id, query.keyword)
-
-    return [
-        TagResponse(
-            id=tag.id,
-            name=tag.name,
-            type=tag.type,
-            binding_count=int(tag.binding_count),
-        )
-        for tag in tags
-    ]
 
 
-@console_router.post(
-    "/tags",
-    response_model=TagResponse,
-    tags=["console"],
-)
-@setup_required
-@login_required
-@account_initialization_required
-def create_tag(payload: TagBasePayload) -> TagResponse:
-    current_user, _ = current_account_with_tenant()
-    # The role of the current user in the tag table must be admin, owner, or editor
-    if not (current_user.has_edit_permission or current_user.is_dataset_editor):
-        raise Forbidden()
+@console_ns.route("/tags")
+class TagListApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @console_ns.doc(
+        params={"type": 'Tag type filter. Can be "knowledge" or "app".', "keyword": "Search keyword for tag name."}
+    )
+    @marshal_with(dataset_tag_fields)
+    def get(self):
+        _, current_tenant_id = current_account_with_tenant()
+        raw_args = request.args.to_dict()
+        param = TagListQueryParam.model_validate(raw_args)
+        tags = TagService.get_tags(param.type, current_tenant_id, param.keyword)
 
-    tag = TagService.save_tags(payload.model_dump())
+        return tags, 200
 
-    return TagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count=0)
+    @console_ns.expect(console_ns.models[TagBasePayload.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        current_user, _ = current_account_with_tenant()
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+            raise Forbidden()
+
+        payload = TagBasePayload.model_validate(console_ns.payload or {})
+        tag = TagService.save_tags(payload.model_dump())
+
+        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0}
+
+        return response, 200
 
 
-@console_router.patch(
-    "/tags/<uuid:tag_id>",
-    response_model=TagResponse,
-    tags=["console"],
-)
-@setup_required
-@login_required
-@account_initialization_required
-def update_tag(tag_id: UUID, payload: TagBasePayload) -> TagResponse:
-    current_user, _ = current_account_with_tenant()
-    tag_id_str = str(tag_id)
-    # The role of the current user in the ta table must be admin, owner, or editor
-    if not (current_user.has_edit_permission or current_user.is_dataset_editor):
-        raise Forbidden()
+@console_ns.route("/tags/<uuid:tag_id>")
+class TagUpdateDeleteApi(Resource):
+    @console_ns.expect(console_ns.models[TagBasePayload.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def patch(self, tag_id):
+        current_user, _ = current_account_with_tenant()
+        tag_id = str(tag_id)
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+            raise Forbidden()
 
-    tag = TagService.update_tags(payload.model_dump(), tag_id_str)
+        payload = TagBasePayload.model_validate(console_ns.payload or {})
+        tag = TagService.update_tags(payload.model_dump(), tag_id)
 
-    binding_count = TagService.get_tag_binding_count(tag_id_str)
+        binding_count = TagService.get_tag_binding_count(tag_id)
 
-    return TagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count=binding_count)
+        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count}
+
+        return response, 200
+
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    def delete(self, tag_id):
+        tag_id = str(tag_id)
+
+        TagService.delete_tag(tag_id)
+
+        return "", 204
 
 
-@console_router.delete(
-    "/tags/<uuid:tag_id>",
-    tags=["console"],
-    status_code=204,
-)
-@setup_required
-@login_required
-@account_initialization_required
-@edit_permission_required
-def delete_tag(tag_id: UUID) -> None:
-    tag_id_str = str(tag_id)
+@console_ns.route("/tag-bindings/create")
+class TagBindingCreateApi(Resource):
+    @console_ns.expect(console_ns.models[TagBindingPayload.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        current_user, _ = current_account_with_tenant()
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+            raise Forbidden()
 
-    TagService.delete_tag(tag_id_str)
+        payload = TagBindingPayload.model_validate(console_ns.payload or {})
+        TagService.save_tag_binding(payload.model_dump())
+
+        return {"result": "success"}, 200
 
 
-@console_router.post(
-    "/tag-bindings/create",
-    response_model=TagBindingResult,
-    tags=["console"],
-)
-@setup_required
-@login_required
-@account_initialization_required
-def create_tag_binding(payload: TagBindingPayload) -> TagBindingResult:
-    current_user, _ = current_account_with_tenant()
-    # The role of the current user in the tag table must be admin, owner, editor, or dataset_operator
-    if not (current_user.has_edit_permission or current_user.is_dataset_editor):
-        raise Forbidden()
+@console_ns.route("/tag-bindings/remove")
+class TagBindingDeleteApi(Resource):
+    @console_ns.expect(console_ns.models[TagBindingRemovePayload.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    def post(self):
+        current_user, _ = current_account_with_tenant()
+        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
+        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+            raise Forbidden()
 
-    TagService.save_tag_binding(payload.model_dump())
+        payload = TagBindingRemovePayload.model_validate(console_ns.payload or {})
+        TagService.delete_tag_binding(payload.model_dump())
 
-    return TagBindingResult(result="success")
-
-
-@console_router.post(
-    "/tag-bindings/remove",
-    response_model=TagBindingResult,
-    tags=["console"],
-)
-@setup_required
-@login_required
-@account_initialization_required
-def delete_tag_binding(payload: TagBindingRemovePayload) -> TagBindingResult:
-    current_user, _ = current_account_with_tenant()
-    # The role of the current user in the tag table must be admin, owner, editor, or dataset_operator
-    if not (current_user.has_edit_permission or current_user.is_dataset_editor):
-        raise Forbidden()
-
-    TagService.delete_tag_binding(payload.model_dump())
-
-    return TagBindingResult(result="success")
+        return {"result": "success"}, 200

api/controllers/console/workspace/account.py

@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
 
 from configs import dify_config
 from constants.languages import supported_language
+from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.auth.error import (
     EmailAlreadyInUseError,
@@ -37,7 +38,7 @@ from controllers.console.wraps import (
     setup_required,
 )
 from extensions.ext_database import db
-from fields.member_fields import account_fields
+from fields.member_fields import Account as AccountResponse
 from libs.datetime_utils import naive_utc_now
 from libs.helper import EmailStr, TimestampField, extract_remote_ip, timezone
 from libs.login import current_account_with_tenant, login_required
@@ -170,6 +171,12 @@ reg(ChangeEmailSendPayload)
 reg(ChangeEmailValidityPayload)
 reg(ChangeEmailResetPayload)
 reg(CheckEmailUniquePayload)
+register_schema_models(console_ns, AccountResponse)
+
+
+def _serialize_account(account) -> dict:
+    return AccountResponse.model_validate(account, from_attributes=True).model_dump(mode="json")
+
 
 integrate_fields = {
     "provider": fields.String,
@@ -236,11 +243,11 @@ class AccountProfileApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_fields)
+    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
     @enterprise_license_required
     def get(self):
         current_user, _ = current_account_with_tenant()
-        return current_user
+        return _serialize_account(current_user)
 
 
 @console_ns.route("/account/name")
@@ -249,14 +256,14 @@ class AccountNameApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_fields)
+    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
         args = AccountNamePayload.model_validate(payload)
         updated_account = AccountService.update_account(current_user, name=args.name)
 
-        return updated_account
+        return _serialize_account(updated_account)
 
 
 @console_ns.route("/account/avatar")
@@ -265,7 +272,7 @@ class AccountAvatarApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_fields)
+    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -273,7 +280,7 @@ class AccountAvatarApi(Resource):
 
         updated_account = AccountService.update_account(current_user, avatar=args.avatar)
 
-        return updated_account
+        return _serialize_account(updated_account)
 
 
 @console_ns.route("/account/interface-language")
@@ -282,7 +289,7 @@ class AccountInterfaceLanguageApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_fields)
+    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -290,7 +297,7 @@ class AccountInterfaceLanguageApi(Resource):
 
         updated_account = AccountService.update_account(current_user, interface_language=args.interface_language)
 
-        return updated_account
+        return _serialize_account(updated_account)
 
 
 @console_ns.route("/account/interface-theme")
@@ -299,7 +306,7 @@ class AccountInterfaceThemeApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_fields)
+    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -307,7 +314,7 @@ class AccountInterfaceThemeApi(Resource):
 
         updated_account = AccountService.update_account(current_user, interface_theme=args.interface_theme)
 
-        return updated_account
+        return _serialize_account(updated_account)
 
 
 @console_ns.route("/account/timezone")
@@ -316,7 +323,7 @@ class AccountTimezoneApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_fields)
+    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -324,7 +331,7 @@ class AccountTimezoneApi(Resource):
 
         updated_account = AccountService.update_account(current_user, timezone=args.timezone)
 
-        return updated_account
+        return _serialize_account(updated_account)
 
 
 @console_ns.route("/account/password")
@@ -333,7 +340,7 @@ class AccountPasswordApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_fields)
+    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -344,7 +351,7 @@ class AccountPasswordApi(Resource):
         except ServiceCurrentPasswordIncorrectError:
             raise CurrentPasswordIncorrectError()
 
-        return {"result": "success"}
+        return _serialize_account(current_user)
 
 
 @console_ns.route("/account/integrates")
@@ -620,7 +627,7 @@ class ChangeEmailResetApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_fields)
+    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
     def post(self):
         payload = console_ns.payload or {}
         args = ChangeEmailResetPayload.model_validate(payload)
@@ -649,7 +656,7 @@ class ChangeEmailResetApi(Resource):
             email=normalized_new_email,
         )
 
-        return updated_account
+        return _serialize_account(updated_account)
 
 
 @console_ns.route("/account/change-email/check-email-unique")

api/controllers/console/workspace/endpoint.py

@@ -1,9 +1,10 @@
 from typing import Any
 
 from flask import request
-from flask_restx import Resource, fields
+from flask_restx import Resource
 from pydantic import BaseModel, Field
 
+from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required
 from core.model_runtime.utils.encoders import jsonable_encoder
@@ -38,15 +39,53 @@ class EndpointListForPluginQuery(EndpointListQuery):
     plugin_id: str
 
 
+class EndpointCreateResponse(BaseModel):
+    success: bool = Field(description="Operation success")
+
+
+class EndpointListResponse(BaseModel):
+    endpoints: list[dict[str, Any]] = Field(description="Endpoint information")
+
+
+class PluginEndpointListResponse(BaseModel):
+    endpoints: list[dict[str, Any]] = Field(description="Endpoint information")
+
+
+class EndpointDeleteResponse(BaseModel):
+    success: bool = Field(description="Operation success")
+
+
+class EndpointUpdateResponse(BaseModel):
+    success: bool = Field(description="Operation success")
+
+
+class EndpointEnableResponse(BaseModel):
+    success: bool = Field(description="Operation success")
+
+
+class EndpointDisableResponse(BaseModel):
+    success: bool = Field(description="Operation success")
+
+
 def reg(cls: type[BaseModel]):
     console_ns.schema_model(cls.__name__, cls.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0))
 
 
-reg(EndpointCreatePayload)
-reg(EndpointIdPayload)
-reg(EndpointUpdatePayload)
-reg(EndpointListQuery)
-reg(EndpointListForPluginQuery)
+register_schema_models(
+    console_ns,
+    EndpointCreatePayload,
+    EndpointIdPayload,
+    EndpointUpdatePayload,
+    EndpointListQuery,
+    EndpointListForPluginQuery,
+    EndpointCreateResponse,
+    EndpointListResponse,
+    PluginEndpointListResponse,
+    EndpointDeleteResponse,
+    EndpointUpdateResponse,
+    EndpointEnableResponse,
+    EndpointDisableResponse,
+)
 
 
 @console_ns.route("/workspaces/current/endpoints/create")
@@ -57,7 +96,7 @@ class EndpointCreateApi(Resource):
     @console_ns.response(
         200,
         "Endpoint created successfully",
-        console_ns.model("EndpointCreateResponse", {"success": fields.Boolean(description="Operation success")}),
+        console_ns.models[EndpointCreateResponse.__name__],
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required
@@ -91,9 +130,7 @@ class EndpointListApi(Resource):
     @console_ns.response(
         200,
         "Success",
-        console_ns.model(
-            "EndpointListResponse", {"endpoints": fields.List(fields.Raw(description="Endpoint information"))}
-        ),
+        console_ns.models[EndpointListResponse.__name__],
     )
     @setup_required
     @login_required
@@ -126,9 +163,7 @@ class EndpointListForSinglePluginApi(Resource):
     @console_ns.response(
         200,
         "Success",
-        console_ns.model(
-            "PluginEndpointListResponse", {"endpoints": fields.List(fields.Raw(description="Endpoint information"))}
-        ),
+        console_ns.models[PluginEndpointListResponse.__name__],
     )
     @setup_required
     @login_required
@@ -163,7 +198,7 @@ class EndpointDeleteApi(Resource):
     @console_ns.response(
         200,
         "Endpoint deleted successfully",
-        console_ns.model("EndpointDeleteResponse", {"success": fields.Boolean(description="Operation success")}),
+        console_ns.models[EndpointDeleteResponse.__name__],
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required
@@ -190,7 +225,7 @@ class EndpointUpdateApi(Resource):
     @console_ns.response(
         200,
         "Endpoint updated successfully",
-        console_ns.model("EndpointUpdateResponse", {"success": fields.Boolean(description="Operation success")}),
+        console_ns.models[EndpointUpdateResponse.__name__],
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required
@@ -221,7 +256,7 @@ class EndpointEnableApi(Resource):
     @console_ns.response(
         200,
         "Endpoint enabled successfully",
-        console_ns.model("EndpointEnableResponse", {"success": fields.Boolean(description="Operation success")}),
+        console_ns.models[EndpointEnableResponse.__name__],
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required
@@ -248,7 +283,7 @@ class EndpointDisableApi(Resource):
     @console_ns.response(
         200,
         "Endpoint disabled successfully",
-        console_ns.model("EndpointDisableResponse", {"success": fields.Boolean(description="Operation success")}),
+        console_ns.models[EndpointDisableResponse.__name__],
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required

api/controllers/console/workspace/members.py

@@ -1,12 +1,12 @@
 from urllib import parse
 
 from flask import abort, request
-from flask_restx import Resource, fields, marshal_with
-from pydantic import BaseModel, Field
+from flask_restx import Resource
+from pydantic import BaseModel, Field, TypeAdapter
 
 import services
 from configs import dify_config
-from controllers.common.schema import get_or_create_model, register_enum_models
+from controllers.common.schema import register_enum_models, register_schema_models
 from controllers.console import console_ns
 from controllers.console.auth.error import (
     CannotTransferOwnerToSelfError,
@@ -25,7 +25,7 @@ from controllers.console.wraps import (
     setup_required,
 )
 from extensions.ext_database import db
-from fields.member_fields import account_with_role_fields, account_with_role_list_fields
+from fields.member_fields import AccountWithRole, AccountWithRoleList
 from libs.helper import extract_remote_ip
 from libs.login import current_account_with_tenant, login_required
 from models.account import Account, TenantAccountRole
@@ -69,12 +69,7 @@ reg(OwnerTransferEmailPayload)
 reg(OwnerTransferCheckPayload)
 reg(OwnerTransferPayload)
 register_enum_models(console_ns, TenantAccountRole)
-
-account_with_role_model = get_or_create_model("AccountWithRole", account_with_role_fields)
-
-account_with_role_list_fields_copy = account_with_role_list_fields.copy()
-account_with_role_list_fields_copy["accounts"] = fields.List(fields.Nested(account_with_role_model))
-account_with_role_list_model = get_or_create_model("AccountWithRoleList", account_with_role_list_fields_copy)
+register_schema_models(console_ns, AccountWithRole, AccountWithRoleList)
 
 
 @console_ns.route("/workspaces/current/members")
@@ -84,13 +79,15 @@ class MemberListApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_with_role_list_model)
+    @console_ns.response(200, "Success", console_ns.models[AccountWithRoleList.__name__])
     def get(self):
         current_user, _ = current_account_with_tenant()
         if not current_user.current_tenant:
             raise ValueError("No current tenant")
         members = TenantService.get_tenant_members(current_user.current_tenant)
-        return {"result": "success", "accounts": members}, 200
+        member_models = TypeAdapter(list[AccountWithRole]).validate_python(members, from_attributes=True)
+        response = AccountWithRoleList(accounts=member_models)
+        return response.model_dump(mode="json"), 200
 
 
 @console_ns.route("/workspaces/current/members/invite-email")
@@ -235,13 +232,15 @@ class DatasetOperatorMemberListApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @marshal_with(account_with_role_list_model)
+    @console_ns.response(200, "Success", console_ns.models[AccountWithRoleList.__name__])
     def get(self):
         current_user, _ = current_account_with_tenant()
         if not current_user.current_tenant:
             raise ValueError("No current tenant")
         members = TenantService.get_dataset_operator_members(current_user.current_tenant)
-        return {"result": "success", "accounts": members}, 200
+        member_models = TypeAdapter(list[AccountWithRole]).validate_python(members, from_attributes=True)
+        response = AccountWithRoleList(accounts=member_models)
+        return response.model_dump(mode="json"), 200
 
 
 @console_ns.route("/workspaces/current/members/send-owner-transfer-confirm-email")

api/controllers/service_api/app/annotation.py

@@ -1,16 +1,16 @@
 from typing import Literal
 
 from flask import request
-from flask_restx import Namespace, Resource, fields
+from flask_restx import Resource
 from flask_restx.api import HTTPStatus
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, TypeAdapter
 
 from controllers.common.schema import register_schema_models
 from controllers.console.wraps import edit_permission_required
 from controllers.service_api import service_api_ns
 from controllers.service_api.wraps import validate_app_token
 from extensions.ext_redis import redis_client
-from fields.annotation_fields import annotation_fields, build_annotation_model
+from fields.annotation_fields import Annotation, AnnotationList
 from models.model import App
 from services.annotation_service import AppAnnotationService
 
@@ -26,7 +26,9 @@ class AnnotationReplyActionPayload(BaseModel):
     embedding_model_name: str = Field(description="Embedding model name")
 
 
-register_schema_models(service_api_ns, AnnotationCreatePayload, AnnotationReplyActionPayload)
+register_schema_models(
+    service_api_ns, AnnotationCreatePayload, AnnotationReplyActionPayload, Annotation, AnnotationList
+)
 
 
 @service_api_ns.route("/apps/annotation-reply/<string:action>")
@@ -45,10 +47,11 @@ class AnnotationReplyActionApi(Resource):
     def post(self, app_model: App, action: Literal["enable", "disable"]):
         """Enable or disable annotation reply feature."""
         args = AnnotationReplyActionPayload.model_validate(service_api_ns.payload or {}).model_dump()
-        if action == "enable":
-            result = AppAnnotationService.enable_app_annotation(args, app_model.id)
-        elif action == "disable":
-            result = AppAnnotationService.disable_app_annotation(app_model.id)
+        match action:
+            case "enable":
+                result = AppAnnotationService.enable_app_annotation(args, app_model.id)
+            case "disable":
+                result = AppAnnotationService.disable_app_annotation(app_model.id)
         return result, 200
 
 
@@ -82,23 +85,6 @@ class AnnotationReplyActionStatusApi(Resource):
         return {"job_id": job_id, "job_status": job_status, "error_msg": error_msg}, 200
 
 
-# Define annotation list response model
-annotation_list_fields = {
-    "data": fields.List(fields.Nested(annotation_fields)),
-    "has_more": fields.Boolean,
-    "limit": fields.Integer,
-    "total": fields.Integer,
-    "page": fields.Integer,
-}
-
-
-def build_annotation_list_model(api_or_ns: Namespace):
-    """Build the annotation list model for the API or Namespace."""
-    copied_annotation_list_fields = annotation_list_fields.copy()
-    copied_annotation_list_fields["data"] = fields.List(fields.Nested(build_annotation_model(api_or_ns)))
-    return api_or_ns.model("AnnotationList", copied_annotation_list_fields)
-
-
 @service_api_ns.route("/apps/annotations")
 class AnnotationListApi(Resource):
     @service_api_ns.doc("list_annotations")
@@ -109,8 +95,12 @@ class AnnotationListApi(Resource):
             401: "Unauthorized - invalid API token",
         }
     )
+    @service_api_ns.response(
+        200,
+        "Annotations retrieved successfully",
+        service_api_ns.models[AnnotationList.__name__],
+    )
     @validate_app_token
-    @service_api_ns.marshal_with(build_annotation_list_model(service_api_ns))
     def get(self, app_model: App):
         """List annotations for the application."""
         page = request.args.get("page", default=1, type=int)
@@ -118,13 +108,15 @@ class AnnotationListApi(Resource):
         keyword = request.args.get("keyword", default="", type=str)
 
         annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id(app_model.id, page, limit, keyword)
-        return {
-            "data": annotation_list,
-            "has_more": len(annotation_list) == limit,
-            "limit": limit,
-            "total": total,
-            "page": page,
-        }
+        annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
+        response = AnnotationList(
+            data=annotation_models,
+            has_more=len(annotation_list) == limit,
+            limit=limit,
+            total=total,
+            page=page,
+        )
+        return response.model_dump(mode="json")
 
     @service_api_ns.expect(service_api_ns.models[AnnotationCreatePayload.__name__])
     @service_api_ns.doc("create_annotation")
@@ -135,13 +127,18 @@ class AnnotationListApi(Resource):
             401: "Unauthorized - invalid API token",
         }
     )
+    @service_api_ns.response(
+        HTTPStatus.CREATED,
+        "Annotation created successfully",
+        service_api_ns.models[Annotation.__name__],
+    )
     @validate_app_token
-    @service_api_ns.marshal_with(build_annotation_model(service_api_ns), code=HTTPStatus.CREATED)
     def post(self, app_model: App):
         """Create a new annotation."""
         args = AnnotationCreatePayload.model_validate(service_api_ns.payload or {}).model_dump()
         annotation = AppAnnotationService.insert_app_annotation_directly(args, app_model.id)
-        return annotation, 201
+        response = Annotation.model_validate(annotation, from_attributes=True)
+        return response.model_dump(mode="json"), HTTPStatus.CREATED
 
 
 @service_api_ns.route("/apps/annotations/<uuid:annotation_id>")
@@ -158,14 +155,19 @@ class AnnotationUpdateDeleteApi(Resource):
             404: "Annotation not found",
         }
     )
+    @service_api_ns.response(
+        200,
+        "Annotation updated successfully",
+        service_api_ns.models[Annotation.__name__],
+    )
     @validate_app_token
     @edit_permission_required
-    @service_api_ns.marshal_with(build_annotation_model(service_api_ns))
     def put(self, app_model: App, annotation_id: str):
         """Update an existing annotation."""
         args = AnnotationCreatePayload.model_validate(service_api_ns.payload or {}).model_dump()
         annotation = AppAnnotationService.update_app_annotation_directly(args, app_model.id, annotation_id)
-        return annotation
+        response = Annotation.model_validate(annotation, from_attributes=True)
+        return response.model_dump(mode="json")
 
     @service_api_ns.doc("delete_annotation")
     @service_api_ns.doc(description="Delete an annotation")

api/controllers/service_api/dataset/dataset.py

@@ -17,7 +17,7 @@ from controllers.service_api.wraps import (
 from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from fields.dataset_fields import dataset_detail_fields
-from fields.tag_fields import build_dataset_tag_fields
+from fields.tag_fields import DataSetTag
 from libs.login import current_user
 from models.account import Account
 from models.dataset import DatasetPermissionEnum
@@ -114,6 +114,7 @@ register_schema_models(
     TagBindingPayload,
     TagUnbindingPayload,
     DatasetListQuery,
+    DataSetTag,
 )
 
 
@@ -395,7 +396,7 @@ class DatasetApi(DatasetApiResource):
         try:
             if DatasetService.delete_dataset(dataset_id_str, current_user):
                 DatasetPermissionService.clear_partial_member_list(dataset_id_str)
-                return 204
+                return "", 204
             else:
                 raise NotFound("Dataset not found.")
         except services.errors.dataset.DatasetInUseError:
@@ -480,15 +481,14 @@ class DatasetTagsApi(DatasetApiResource):
             401: "Unauthorized - invalid API token",
         }
     )
-    @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
     def get(self, _):
         """Get all knowledge type tags."""
         assert isinstance(current_user, Account)
         cid = current_user.current_tenant_id
         assert cid is not None
         tags = TagService.get_tags("knowledge", cid)
-
-        return tags, 200
+        tag_models = TypeAdapter(list[DataSetTag]).validate_python(tags, from_attributes=True)
+        return [tag.model_dump(mode="json") for tag in tag_models], 200
 
     @service_api_ns.expect(service_api_ns.models[TagCreatePayload.__name__])
     @service_api_ns.doc("create_dataset_tag")
@@ -500,7 +500,6 @@ class DatasetTagsApi(DatasetApiResource):
             403: "Forbidden - insufficient permissions",
         }
     )
-    @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
     def post(self, _):
         """Add a knowledge type tag."""
         assert isinstance(current_user, Account)
@@ -510,7 +509,9 @@ class DatasetTagsApi(DatasetApiResource):
         payload = TagCreatePayload.model_validate(service_api_ns.payload or {})
         tag = TagService.save_tags({"name": payload.name, "type": "knowledge"})
 
-        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0}
+        response = DataSetTag.model_validate(
+            {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0}
+        ).model_dump(mode="json")
         return response, 200
 
     @service_api_ns.expect(service_api_ns.models[TagUpdatePayload.__name__])
@@ -523,7 +524,6 @@ class DatasetTagsApi(DatasetApiResource):
             403: "Forbidden - insufficient permissions",
         }
     )
-    @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
     def patch(self, _):
         assert isinstance(current_user, Account)
         if not (current_user.has_edit_permission or current_user.is_dataset_editor):
@@ -536,8 +536,9 @@ class DatasetTagsApi(DatasetApiResource):
 
         binding_count = TagService.get_tag_binding_count(tag_id)
 
-        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count}
-
+        response = DataSetTag.model_validate(
+            {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count}
+        ).model_dump(mode="json")
         return response, 200
 
     @service_api_ns.expect(service_api_ns.models[TagDeletePayload.__name__])
@@ -556,7 +557,7 @@ class DatasetTagsApi(DatasetApiResource):
         payload = TagDeletePayload.model_validate(service_api_ns.payload or {})
         TagService.delete_tag(payload.tag_id)
 
-        return 204
+        return "", 204
 
 
 @service_api_ns.route("/datasets/tags/binding")
@@ -580,7 +581,7 @@ class DatasetTagBindingApi(DatasetApiResource):
         payload = TagBindingPayload.model_validate(service_api_ns.payload or {})
         TagService.save_tag_binding({"tag_ids": payload.tag_ids, "target_id": payload.target_id, "type": "knowledge"})
 
-        return 204
+        return "", 204
 
 
 @service_api_ns.route("/datasets/tags/unbinding")
@@ -604,7 +605,7 @@ class DatasetTagUnbindingApi(DatasetApiResource):
         payload = TagUnbindingPayload.model_validate(service_api_ns.payload or {})
         TagService.delete_tag_binding({"tag_id": payload.tag_id, "target_id": payload.target_id, "type": "knowledge"})
 
-        return 204
+        return "", 204
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/tags")

api/controllers/service_api/dataset/document.py

@@ -746,4 +746,4 @@ class DocumentApi(DatasetApiResource):
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError("Cannot delete document during indexing.")
 
-        return 204
+        return "", 204

api/controllers/service_api/dataset/metadata.py

@@ -128,7 +128,7 @@ class DatasetMetadataServiceApi(DatasetApiResource):
         DatasetService.check_dataset_permission(dataset, current_user)
 
         MetadataService.delete_metadata(dataset_id_str, metadata_id_str)
-        return 204
+        return "", 204
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/metadata/built-in")
@@ -168,10 +168,11 @@ class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource):
             raise NotFound("Dataset not found.")
         DatasetService.check_dataset_permission(dataset, current_user)
 
-        if action == "enable":
-            MetadataService.enable_built_in_field(dataset)
-        elif action == "disable":
-            MetadataService.disable_built_in_field(dataset)
+        match action:
+            case "enable":
+                MetadataService.enable_built_in_field(dataset)
+            case "disable":
+                MetadataService.disable_built_in_field(dataset)
         return {"result": "success"}, 200
 
 

api/controllers/service_api/dataset/segment.py

@@ -233,7 +233,7 @@ class DatasetSegmentApi(DatasetApiResource):
         if not segment:
             raise NotFound("Segment not found.")
         SegmentService.delete_segment(segment, document, dataset)
-        return 204
+        return "", 204
 
     @service_api_ns.expect(service_api_ns.models[SegmentUpdatePayload.__name__])
     @service_api_ns.doc("update_segment")
@@ -499,7 +499,7 @@ class DatasetChildChunkApi(DatasetApiResource):
         except ChildChunkDeleteIndexServiceError as e:
             raise ChildChunkDeleteIndexError(str(e))
 
-        return 204
+        return "", 204
 
     @service_api_ns.expect(service_api_ns.models[ChildChunkUpdatePayload.__name__])
     @service_api_ns.doc("update_child_chunk")

api/controllers/service_api/wraps.py

@@ -1,27 +1,24 @@
 import logging
 import time
 from collections.abc import Callable
-from datetime import timedelta
 from enum import StrEnum, auto
 from functools import wraps
-from typing import Concatenate, ParamSpec, TypeVar
+from typing import Concatenate, ParamSpec, TypeVar, cast
 
 from flask import current_app, request
 from flask_login import user_logged_in
 from flask_restx import Resource
 from pydantic import BaseModel
-from sqlalchemy import select, update
-from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden, NotFound, Unauthorized
 
 from enums.cloud_plan import CloudPlan
 from extensions.ext_database import db
 from extensions.ext_redis import redis_client
-from libs.datetime_utils import naive_utc_now
 from libs.login import current_user
 from models import Account, Tenant, TenantAccountJoin, TenantStatus
 from models.dataset import Dataset, RateLimitLog
 from models.model import ApiToken, App
+from services.api_token_service import ApiTokenCache, fetch_token_with_single_flight, record_token_usage
 from services.end_user_service import EndUserService
 from services.feature_service import FeatureService
 
@@ -73,14 +70,14 @@ def validate_app_token(view: Callable[P, R] | None = None, *, fetch_user_arg: Fe
 
             # If caller needs end-user context, attach EndUser to current_user
             if fetch_user_arg:
-                if fetch_user_arg.fetch_from == WhereisUserArg.QUERY:
-                    user_id = request.args.get("user")
-                elif fetch_user_arg.fetch_from == WhereisUserArg.JSON:
-                    user_id = request.get_json().get("user")
-                elif fetch_user_arg.fetch_from == WhereisUserArg.FORM:
-                    user_id = request.form.get("user")
-                else:
-                    user_id = None
+                user_id = None
+                match fetch_user_arg.fetch_from:
+                    case WhereisUserArg.QUERY:
+                        user_id = request.args.get("user")
+                    case WhereisUserArg.JSON:
+                        user_id = request.get_json().get("user")
+                    case WhereisUserArg.FORM:
+                        user_id = request.form.get("user")
 
                 if not user_id and fetch_user_arg.required:
                     raise ValueError("Arg user must be provided.")
@@ -296,7 +293,14 @@ def validate_dataset_token(view: Callable[Concatenate[T, P], R] | None = None):
 
 def validate_and_get_api_token(scope: str | None = None):
     """
-    Validate and get API token.
+    Validate and get API token with Redis caching.
+
+    This function uses a two-tier approach:
+    1. First checks Redis cache for the token
+    2. If not cached, queries database and caches the result
+
+    The last_used_at field is updated asynchronously via Celery task
+    to avoid blocking the request.
     """
     auth_header = request.headers.get("Authorization")
     if auth_header is None or " " not in auth_header:
@@ -308,29 +312,18 @@ def validate_and_get_api_token(scope: str | None = None):
     if auth_scheme != "bearer":
         raise Unauthorized("Authorization scheme must be 'Bearer'")
 
-    current_time = naive_utc_now()
-    cutoff_time = current_time - timedelta(minutes=1)
-    with Session(db.engine, expire_on_commit=False) as session:
-        update_stmt = (
-            update(ApiToken)
-            .where(
-                ApiToken.token == auth_token,
-                (ApiToken.last_used_at.is_(None) | (ApiToken.last_used_at < cutoff_time)),
-                ApiToken.type == scope,
-            )
-            .values(last_used_at=current_time)
-        )
-        stmt = select(ApiToken).where(ApiToken.token == auth_token, ApiToken.type == scope)
-        result = session.execute(update_stmt)
-        api_token = session.scalar(stmt)
+    # Try to get token from cache first
+    # Returns a CachedApiToken (plain Python object), not a SQLAlchemy model
+    cached_token = ApiTokenCache.get(auth_token, scope)
+    if cached_token is not None:
+        logger.debug("Token validation served from cache for scope: %s", scope)
+        # Record usage in Redis for later batch update (no Celery task per request)
+        record_token_usage(auth_token, scope)
+        return cast(ApiToken, cached_token)
 
-        if hasattr(result, "rowcount") and result.rowcount > 0:
-            session.commit()
-
-        if not api_token:
-            raise Unauthorized("Access token is invalid")
-
-    return api_token
+    # Cache miss - use Redis lock for single-flight mode
+    # This ensures only one request queries DB for the same token concurrently
+    return fetch_token_with_single_flight(auth_token, scope)
 
 
 class DatasetApiResource(Resource):

api/core/app/app_config/easy_ui_based_app/agent/manager.py

@@ -14,16 +14,17 @@ class AgentConfigManager:
             agent_dict = config.get("agent_mode", {})
             agent_strategy = agent_dict.get("strategy", "cot")
 
-            if agent_strategy == "function_call":
-                strategy = AgentEntity.Strategy.FUNCTION_CALLING
-            elif agent_strategy in {"cot", "react"}:
-                strategy = AgentEntity.Strategy.CHAIN_OF_THOUGHT
-            else:
-                # old configs, try to detect default strategy
-                if config["model"]["provider"] == "openai":
+            match agent_strategy:
+                case "function_call":
                     strategy = AgentEntity.Strategy.FUNCTION_CALLING
-                else:
+                case "cot" | "react":
                     strategy = AgentEntity.Strategy.CHAIN_OF_THOUGHT
+                case _:
+                    # old configs, try to detect default strategy
+                    if config["model"]["provider"] == "openai":
+                        strategy = AgentEntity.Strategy.FUNCTION_CALLING
+                    else:
+                        strategy = AgentEntity.Strategy.CHAIN_OF_THOUGHT
 
             agent_tools = []
             for tool in agent_dict.get("tools", []):

api/core/app/apps/common/workflow_response_converter.py

@@ -250,7 +250,7 @@ class WorkflowResponseConverter:
             data=WorkflowFinishStreamResponse.Data(
                 id=run_id,
                 workflow_id=workflow_id,
-                status=status.value,
+                status=status,
                 outputs=encoded_outputs,
                 error=error,
                 elapsed_time=elapsed_time,
@@ -340,13 +340,13 @@ class WorkflowResponseConverter:
         metadata = self._merge_metadata(event.execution_metadata, snapshot)
 
         if isinstance(event, QueueNodeSucceededEvent):
-            status = WorkflowNodeExecutionStatus.SUCCEEDED.value
+            status = WorkflowNodeExecutionStatus.SUCCEEDED
             error_message = event.error
         elif isinstance(event, QueueNodeFailedEvent):
-            status = WorkflowNodeExecutionStatus.FAILED.value
+            status = WorkflowNodeExecutionStatus.FAILED
             error_message = event.error
         else:
-            status = WorkflowNodeExecutionStatus.EXCEPTION.value
+            status = WorkflowNodeExecutionStatus.EXCEPTION
             error_message = event.error
 
         return NodeFinishStreamResponse(
@@ -413,7 +413,7 @@ class WorkflowResponseConverter:
                 process_data_truncated=process_data_truncated,
                 outputs=outputs,
                 outputs_truncated=outputs_truncated,
-                status=WorkflowNodeExecutionStatus.RETRY.value,
+                status=WorkflowNodeExecutionStatus.RETRY,
                 error=event.error,
                 elapsed_time=elapsed_time,
                 execution_metadata=metadata,

api/core/app/apps/pipeline/pipeline_generator.py

@@ -120,7 +120,7 @@ class PipelineGenerator(BaseAppGenerator):
                 raise ValueError("Pipeline dataset is required")
         inputs: Mapping[str, Any] = args["inputs"]
         start_node_id: str = args["start_node_id"]
-        datasource_type: str = args["datasource_type"]
+        datasource_type = DatasourceProviderType(args["datasource_type"])
         datasource_info_list: list[Mapping[str, Any]] = self._format_datasource_info_list(
             datasource_type, args["datasource_info_list"], pipeline, workflow, start_node_id, user
         )
@@ -660,7 +660,7 @@ class PipelineGenerator(BaseAppGenerator):
         tenant_id: str,
         dataset_id: str,
         built_in_field_enabled: bool,
-        datasource_type: str,
+        datasource_type: DatasourceProviderType,
         datasource_info: Mapping[str, Any],
         created_from: str,
         position: int,
@@ -668,17 +668,17 @@ class PipelineGenerator(BaseAppGenerator):
         batch: str,
         document_form: str,
     ):
-        if datasource_type == "local_file":
-            name = datasource_info.get("name", "untitled")
-        elif datasource_type == "online_document":
-            name = datasource_info.get("page", {}).get("page_name", "untitled")
-        elif datasource_type == "website_crawl":
-            name = datasource_info.get("title", "untitled")
-        elif datasource_type == "online_drive":
-            name = datasource_info.get("name", "untitled")
-        else:
-            raise ValueError(f"Unsupported datasource type: {datasource_type}")
-
+        match datasource_type:
+            case DatasourceProviderType.LOCAL_FILE:
+                name = datasource_info.get("name", "untitled")
+            case DatasourceProviderType.ONLINE_DOCUMENT:
+                name = datasource_info.get("page", {}).get("page_name", "untitled")
+            case DatasourceProviderType.WEBSITE_CRAWL:
+                name = datasource_info.get("title", "untitled")
+            case DatasourceProviderType.ONLINE_DRIVE:
+                name = datasource_info.get("name", "untitled")
+            case _:
+                raise ValueError(f"Unsupported datasource type: {datasource_type}")
         document = Document(
             tenant_id=tenant_id,
             dataset_id=dataset_id,
@@ -706,7 +706,7 @@ class PipelineGenerator(BaseAppGenerator):
 
     def _format_datasource_info_list(
         self,
-        datasource_type: str,
+        datasource_type: DatasourceProviderType,
         datasource_info_list: list[Mapping[str, Any]],
         pipeline: Pipeline,
         workflow: Workflow,
@@ -716,7 +716,7 @@ class PipelineGenerator(BaseAppGenerator):
         """
         Format datasource info list.
         """
-        if datasource_type == "online_drive":
+        if datasource_type == DatasourceProviderType.ONLINE_DRIVE:
             all_files: list[Mapping[str, Any]] = []
             datasource_node_data = None
             datasource_nodes = workflow.graph_dict.get("nodes", [])

api/core/app/entities/task_entities.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel, ConfigDict, Field
 from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage
 from core.rag.entities.citation_metadata import RetrievalSourceMetadata
 from core.workflow.entities import AgentNodeStrategyInit
-from core.workflow.enums import WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
+from core.workflow.enums import WorkflowExecutionStatus, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
 
 
 class AnnotationReplyAccount(BaseModel):
@@ -223,7 +223,7 @@ class WorkflowFinishStreamResponse(StreamResponse):
 
         id: str
         workflow_id: str
-        status: str
+        status: WorkflowExecutionStatus
         outputs: Mapping[str, Any] | None = None
         error: str | None = None
         elapsed_time: float
@@ -311,7 +311,7 @@ class NodeFinishStreamResponse(StreamResponse):
         process_data_truncated: bool = False
         outputs: Mapping[str, Any] | None = None
         outputs_truncated: bool = True
-        status: str
+        status: WorkflowNodeExecutionStatus
         error: str | None = None
         elapsed_time: float
         execution_metadata: Mapping[WorkflowNodeExecutionMetadataKey, Any] | None = None
@@ -375,7 +375,7 @@ class NodeRetryStreamResponse(StreamResponse):
         process_data_truncated: bool = False
         outputs: Mapping[str, Any] | None = None
         outputs_truncated: bool = False
-        status: str
+        status: WorkflowNodeExecutionStatus
         error: str | None = None
         elapsed_time: float
         execution_metadata: Mapping[WorkflowNodeExecutionMetadataKey, Any] | None = None
@@ -719,7 +719,7 @@ class WorkflowAppBlockingResponse(AppBlockingResponse):
 
         id: str
         workflow_id: str
-        status: str
+        status: WorkflowExecutionStatus
         outputs: Mapping[str, Any] | None = None
         error: str | None = None
         elapsed_time: float

api/core/app/workflow/node_factory.py

@@ -47,6 +47,7 @@ class DifyNodeFactory(NodeFactory):
         code_providers: Sequence[type[CodeNodeProvider]] | None = None,
         code_limits: CodeNodeLimits | None = None,
         template_renderer: Jinja2TemplateRenderer | None = None,
+        template_transform_max_output_length: int | None = None,
         http_request_http_client: HttpClientProtocol | None = None,
         http_request_tool_file_manager_factory: Callable[[], ToolFileManager] = ToolFileManager,
         http_request_file_manager: FileManagerProtocol | None = None,
@@ -68,6 +69,9 @@ class DifyNodeFactory(NodeFactory):
             max_object_array_length=dify_config.CODE_MAX_OBJECT_ARRAY_LENGTH,
         )
         self._template_renderer = template_renderer or CodeExecutorJinja2TemplateRenderer()
+        self._template_transform_max_output_length = (
+            template_transform_max_output_length or dify_config.TEMPLATE_TRANSFORM_MAX_LENGTH
+        )
         self._http_request_http_client = http_request_http_client or ssrf_proxy
         self._http_request_tool_file_manager_factory = http_request_tool_file_manager_factory
         self._http_request_file_manager = http_request_file_manager or file_manager
@@ -122,6 +126,7 @@ class DifyNodeFactory(NodeFactory):
                 graph_init_params=self.graph_init_params,
                 graph_runtime_state=self.graph_runtime_state,
                 template_renderer=self._template_renderer,
+                max_output_length=self._template_transform_max_output_length,
             )
 
         if node_type == NodeType.HTTP_REQUEST:

api/core/helper/marketplace.py

@@ -6,7 +6,8 @@ from yarl import URL
 
 from configs import dify_config
 from core.helper.download import download_with_size_limit
-from core.plugin.entities.marketplace import MarketplacePluginDeclaration
+from core.plugin.entities.marketplace import MarketplacePluginDeclaration, MarketplacePluginSnapshot
+from extensions.ext_redis import redis_client
 
 marketplace_api_url = URL(str(dify_config.MARKETPLACE_API_URL))
 logger = logging.getLogger(__name__)
@@ -43,28 +44,37 @@ def batch_fetch_plugin_by_ids(plugin_ids: list[str]) -> list[dict]:
     return data.get("data", {}).get("plugins", [])
 
 
-def batch_fetch_plugin_manifests_ignore_deserialization_error(
-    plugin_ids: list[str],
-) -> Sequence[MarketplacePluginDeclaration]:
-    if len(plugin_ids) == 0:
-        return []
-
-    url = str(marketplace_api_url / "api/v1/plugins/batch")
-    response = httpx.post(url, json={"plugin_ids": plugin_ids}, headers={"X-Dify-Version": dify_config.project.version})
-    response.raise_for_status()
-    result: list[MarketplacePluginDeclaration] = []
-    for plugin in response.json()["data"]["plugins"]:
-        try:
-            result.append(MarketplacePluginDeclaration.model_validate(plugin))
-        except Exception:
-            logger.exception(
-                "Failed to deserialize marketplace plugin manifest for %s", plugin.get("plugin_id", "unknown")
-            )
-
-    return result
-
-
 def record_install_plugin_event(plugin_unique_identifier: str):
     url = str(marketplace_api_url / "api/v1/stats/plugins/install_count")
     response = httpx.post(url, json={"unique_identifier": plugin_unique_identifier})
     response.raise_for_status()
+
+
+def fetch_global_plugin_manifest(cache_key_prefix: str, cache_ttl: int) -> None:
+    """
+    Fetch all plugin manifests from marketplace and cache them in Redis.
+    This should be called once per check cycle to populate the instance-level cache.
+
+    Args:
+        cache_key_prefix: Redis key prefix for caching plugin manifests
+        cache_ttl: Cache TTL in seconds
+
+    Raises:
+        httpx.HTTPError: If the HTTP request fails
+        Exception: If any other error occurs during fetching or caching
+    """
+    url = str(marketplace_api_url / "api/v1/dist/plugins/manifest.json")
+    response = httpx.get(url, headers={"X-Dify-Version": dify_config.project.version}, timeout=30)
+    response.raise_for_status()
+
+    raw_json = response.json()
+    plugins_data = raw_json.get("plugins", [])
+
+    # Parse and cache all plugin snapshots
+    for plugin_data in plugins_data:
+        plugin_snapshot = MarketplacePluginSnapshot.model_validate(plugin_data)
+        redis_client.setex(
+            name=f"{cache_key_prefix}{plugin_snapshot.plugin_id}",
+            time=cache_ttl,
+            value=plugin_snapshot.model_dump_json(),
+        )

api/core/indexing_runner.py

@@ -369,77 +369,78 @@ class IndexingRunner:
         # Generate summary preview
         summary_index_setting = tmp_processing_rule.get("summary_index_setting")
         if summary_index_setting and summary_index_setting.get("enable") and preview_texts:
-            preview_texts = index_processor.generate_summary_preview(tenant_id, preview_texts, summary_index_setting)
+            preview_texts = index_processor.generate_summary_preview(
+                tenant_id, preview_texts, summary_index_setting, doc_language
+            )
 
         return IndexingEstimate(total_segments=total_segments, preview=preview_texts)
 
     def _extract(
         self, index_processor: BaseIndexProcessor, dataset_document: DatasetDocument, process_rule: dict
     ) -> list[Document]:
-        # load file
-        if dataset_document.data_source_type not in {"upload_file", "notion_import", "website_crawl"}:
-            return []
-
         data_source_info = dataset_document.data_source_info_dict
         text_docs = []
-        if dataset_document.data_source_type == "upload_file":
-            if not data_source_info or "upload_file_id" not in data_source_info:
-                raise ValueError("no upload file found")
-            stmt = select(UploadFile).where(UploadFile.id == data_source_info["upload_file_id"])
-            file_detail = db.session.scalars(stmt).one_or_none()
+        match dataset_document.data_source_type:
+            case "upload_file":
+                if not data_source_info or "upload_file_id" not in data_source_info:
+                    raise ValueError("no upload file found")
+                stmt = select(UploadFile).where(UploadFile.id == data_source_info["upload_file_id"])
+                file_detail = db.session.scalars(stmt).one_or_none()
 
-            if file_detail:
+                if file_detail:
+                    extract_setting = ExtractSetting(
+                        datasource_type=DatasourceType.FILE,
+                        upload_file=file_detail,
+                        document_model=dataset_document.doc_form,
+                    )
+                    text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
+            case "notion_import":
+                if (
+                    not data_source_info
+                    or "notion_workspace_id" not in data_source_info
+                    or "notion_page_id" not in data_source_info
+                ):
+                    raise ValueError("no notion import info found")
                 extract_setting = ExtractSetting(
-                    datasource_type=DatasourceType.FILE,
-                    upload_file=file_detail,
+                    datasource_type=DatasourceType.NOTION,
+                    notion_info=NotionInfo.model_validate(
+                        {
+                            "credential_id": data_source_info.get("credential_id"),
+                            "notion_workspace_id": data_source_info["notion_workspace_id"],
+                            "notion_obj_id": data_source_info["notion_page_id"],
+                            "notion_page_type": data_source_info["type"],
+                            "document": dataset_document,
+                            "tenant_id": dataset_document.tenant_id,
+                        }
+                    ),
                     document_model=dataset_document.doc_form,
                 )
                 text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
-        elif dataset_document.data_source_type == "notion_import":
-            if (
-                not data_source_info
-                or "notion_workspace_id" not in data_source_info
-                or "notion_page_id" not in data_source_info
-            ):
-                raise ValueError("no notion import info found")
-            extract_setting = ExtractSetting(
-                datasource_type=DatasourceType.NOTION,
-                notion_info=NotionInfo.model_validate(
-                    {
-                        "credential_id": data_source_info.get("credential_id"),
-                        "notion_workspace_id": data_source_info["notion_workspace_id"],
-                        "notion_obj_id": data_source_info["notion_page_id"],
-                        "notion_page_type": data_source_info["type"],
-                        "document": dataset_document,
-                        "tenant_id": dataset_document.tenant_id,
-                    }
-                ),
-                document_model=dataset_document.doc_form,
-            )
-            text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
-        elif dataset_document.data_source_type == "website_crawl":
-            if (
-                not data_source_info
-                or "provider" not in data_source_info
-                or "url" not in data_source_info
-                or "job_id" not in data_source_info
-            ):
-                raise ValueError("no website import info found")
-            extract_setting = ExtractSetting(
-                datasource_type=DatasourceType.WEBSITE,
-                website_info=WebsiteInfo.model_validate(
-                    {
-                        "provider": data_source_info["provider"],
-                        "job_id": data_source_info["job_id"],
-                        "tenant_id": dataset_document.tenant_id,
-                        "url": data_source_info["url"],
-                        "mode": data_source_info["mode"],
-                        "only_main_content": data_source_info["only_main_content"],
-                    }
-                ),
-                document_model=dataset_document.doc_form,
-            )
-            text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
+            case "website_crawl":
+                if (
+                    not data_source_info
+                    or "provider" not in data_source_info
+                    or "url" not in data_source_info
+                    or "job_id" not in data_source_info
+                ):
+                    raise ValueError("no website import info found")
+                extract_setting = ExtractSetting(
+                    datasource_type=DatasourceType.WEBSITE,
+                    website_info=WebsiteInfo.model_validate(
+                        {
+                            "provider": data_source_info["provider"],
+                            "job_id": data_source_info["job_id"],
+                            "tenant_id": dataset_document.tenant_id,
+                            "url": data_source_info["url"],
+                            "mode": data_source_info["mode"],
+                            "only_main_content": data_source_info["only_main_content"],
+                        }
+                    ),
+                    document_model=dataset_document.doc_form,
+                )
+                text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
+            case _:
+                return []
         # update document status to splitting
         self._update_document_index_status(
             document_id=dataset_document.id,

api/core/llm_generator/prompts.py

@@ -441,11 +441,13 @@ DEFAULT_GENERATOR_SUMMARY_PROMPT = (
 
 Requirements:
 1. Write a concise summary in plain text
-2. Use the same language as the input content
+2. You must write in {language}. No language other than {language} should be used.
 3. Focus on important facts, concepts, and details
 4. If images are included, describe their key information
 5. Do not use words like "好的", "ok", "I understand", "This text discusses", "The content mentions"
 6. Write directly without extra words
+7. If there is not enough content to generate a meaningful summary, 
+   return an empty string without any explanation or prompt
 
 Output only the summary text. Start summarizing now:
 

api/core/plugin/entities/marketplace.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel, Field, computed_field, model_validator
 
 from core.model_runtime.entities.provider_entities import ProviderEntity
 from core.plugin.entities.endpoint import EndpointProviderDeclaration
@@ -48,3 +48,15 @@ class MarketplacePluginDeclaration(BaseModel):
         if "tool" in data and not data["tool"]:
             del data["tool"]
         return data
+
+
+class MarketplacePluginSnapshot(BaseModel):
+    org: str
+    name: str
+    latest_version: str
+    latest_package_identifier: str
+    latest_package_url: str
+
+    @computed_field
+    def plugin_id(self) -> str:
+        return f"{self.org}/{self.name}"

api/core/rag/index_processor/index_processor_base.py

@@ -48,12 +48,22 @@ class BaseIndexProcessor(ABC):
 
     @abstractmethod
     def generate_summary_preview(
-        self, tenant_id: str, preview_texts: list[PreviewDetail], summary_index_setting: dict
+        self,
+        tenant_id: str,
+        preview_texts: list[PreviewDetail],
+        summary_index_setting: dict,
+        doc_language: str | None = None,
     ) -> list[PreviewDetail]:
         """
         For each segment in preview_texts, generate a summary using LLM and attach it to the segment.
         The summary can be stored in a new attribute, e.g., summary.
         This method should be implemented by subclasses.
+
+        Args:
+            tenant_id: Tenant ID
+            preview_texts: List of preview details to generate summaries for
+            summary_index_setting: Summary index configuration
+            doc_language: Optional document language to ensure summary is generated in the correct language
         """
         raise NotImplementedError
 

api/core/rag/index_processor/processor/paragraph_index_processor.py

@@ -275,7 +275,11 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
             raise ValueError("Chunks is not a list")
 
     def generate_summary_preview(
-        self, tenant_id: str, preview_texts: list[PreviewDetail], summary_index_setting: dict
+        self,
+        tenant_id: str,
+        preview_texts: list[PreviewDetail],
+        summary_index_setting: dict,
+        doc_language: str | None = None,
     ) -> list[PreviewDetail]:
         """
         For each segment, concurrently call generate_summary to generate a summary
@@ -298,11 +302,15 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
             if flask_app:
                 # Ensure Flask app context in worker thread
                 with flask_app.app_context():
-                    summary, _ = self.generate_summary(tenant_id, preview.content, summary_index_setting)
+                    summary, _ = self.generate_summary(
+                        tenant_id, preview.content, summary_index_setting, document_language=doc_language
+                    )
                     preview.summary = summary
             else:
                 # Fallback: try without app context (may fail)
-                summary, _ = self.generate_summary(tenant_id, preview.content, summary_index_setting)
+                summary, _ = self.generate_summary(
+                    tenant_id, preview.content, summary_index_setting, document_language=doc_language
+                )
                 preview.summary = summary
 
         # Generate summaries concurrently using ThreadPoolExecutor
@@ -356,6 +364,7 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
         text: str,
         summary_index_setting: dict | None = None,
         segment_id: str | None = None,
+        document_language: str | None = None,
     ) -> tuple[str, LLMUsage]:
         """
         Generate summary for the given text using ModelInstance.invoke_llm and the default or custom summary prompt,
@@ -366,6 +375,8 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
             text: Text content to summarize
             summary_index_setting: Summary index configuration
             segment_id: Optional segment ID to fetch attachments from SegmentAttachmentBinding table
+            document_language: Optional document language (e.g., "Chinese", "English")
+                to ensure summary is generated in the correct language
 
         Returns:
             Tuple of (summary_content, llm_usage) where llm_usage is LLMUsage object
@@ -381,8 +392,22 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
             raise ValueError("model_name and model_provider_name are required in summary_index_setting")
 
         # Import default summary prompt
+        is_default_prompt = False
         if not summary_prompt:
             summary_prompt = DEFAULT_GENERATOR_SUMMARY_PROMPT
+            is_default_prompt = True
+
+        # Format prompt with document language only for default prompt
+        # Custom prompts are used as-is to avoid interfering with user-defined templates
+        # If document_language is provided, use it; otherwise, use "the same language as the input content"
+        # This is especially important for image-only chunks where text is empty or minimal
+        if is_default_prompt:
+            language_for_prompt = document_language or "the same language as the input content"
+            try:
+                summary_prompt = summary_prompt.format(language=language_for_prompt)
+            except KeyError:
+                # If default prompt doesn't have {language} placeholder, use it as-is
+                pass
 
         provider_manager = ProviderManager()
         provider_model_bundle = provider_manager.get_provider_model_bundle(

api/core/rag/index_processor/processor/parent_child_index_processor.py

@@ -358,7 +358,11 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
         }
 
     def generate_summary_preview(
-        self, tenant_id: str, preview_texts: list[PreviewDetail], summary_index_setting: dict
+        self,
+        tenant_id: str,
+        preview_texts: list[PreviewDetail],
+        summary_index_setting: dict,
+        doc_language: str | None = None,
     ) -> list[PreviewDetail]:
         """
         For each parent chunk in preview_texts, concurrently call generate_summary to generate a summary
@@ -389,6 +393,7 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
                         tenant_id=tenant_id,
                         text=preview.content,
                         summary_index_setting=summary_index_setting,
+                        document_language=doc_language,
                     )
                     preview.summary = summary
             else:
@@ -397,6 +402,7 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
                     tenant_id=tenant_id,
                     text=preview.content,
                     summary_index_setting=summary_index_setting,
+                    document_language=doc_language,
                 )
                 preview.summary = summary
 

api/core/rag/index_processor/processor/qa_index_processor.py

@@ -241,7 +241,11 @@ class QAIndexProcessor(BaseIndexProcessor):
         }
 
     def generate_summary_preview(
-        self, tenant_id: str, preview_texts: list[PreviewDetail], summary_index_setting: dict
+        self,
+        tenant_id: str,
+        preview_texts: list[PreviewDetail],
+        summary_index_setting: dict,
+        doc_language: str | None = None,
     ) -> list[PreviewDetail]:
         """
         QA model doesn't generate summaries, so this method returns preview_texts unchanged.

api/core/variables/variables.py

@@ -112,7 +112,7 @@ class ArrayBooleanVariable(ArrayBooleanSegment, ArrayVariable):
 
 class RAGPipelineVariable(BaseModel):
     belong_to_node_id: str = Field(description="belong to which node id, shared means public")
-    type: str = Field(description="variable type, text-input, paragraph, select, number,  file, file-list")
+    type: str = Field(description="variable type, text-input, paragraph, select, number, file, file-list")
     label: str = Field(description="label")
     description: str | None = Field(description="description", default="")
     variable: str = Field(description="variable key", default="")

api/core/workflow/nodes/agent/agent_node.py

@@ -192,32 +192,33 @@ class AgentNode(Node[AgentNodeData]):
                 result[parameter_name] = None
                 continue
             agent_input = node_data.agent_parameters[parameter_name]
-            if agent_input.type == "variable":
-                variable = variable_pool.get(agent_input.value)  # type: ignore
-                if variable is None:
-                    raise AgentVariableNotFoundError(str(agent_input.value))
-                parameter_value = variable.value
-            elif agent_input.type in {"mixed", "constant"}:
-                # variable_pool.convert_template expects a string template,
-                # but if passing a dict, convert to JSON string first before rendering
-                try:
-                    if not isinstance(agent_input.value, str):
-                        parameter_value = json.dumps(agent_input.value, ensure_ascii=False)
-                    else:
+            match agent_input.type:
+                case "variable":
+                    variable = variable_pool.get(agent_input.value)  # type: ignore
+                    if variable is None:
+                        raise AgentVariableNotFoundError(str(agent_input.value))
+                    parameter_value = variable.value
+                case "mixed" | "constant":
+                    # variable_pool.convert_template expects a string template,
+                    # but if passing a dict, convert to JSON string first before rendering
+                    try:
+                        if not isinstance(agent_input.value, str):
+                            parameter_value = json.dumps(agent_input.value, ensure_ascii=False)
+                        else:
+                            parameter_value = str(agent_input.value)
+                    except TypeError:
                         parameter_value = str(agent_input.value)
-                except TypeError:
-                    parameter_value = str(agent_input.value)
-                segment_group = variable_pool.convert_template(parameter_value)
-                parameter_value = segment_group.log if for_log else segment_group.text
-                # variable_pool.convert_template returns a string,
-                # so we need to convert it back to a dictionary
-                try:
-                    if not isinstance(agent_input.value, str):
-                        parameter_value = json.loads(parameter_value)
-                except json.JSONDecodeError:
-                    parameter_value = parameter_value
-            else:
-                raise AgentInputTypeError(agent_input.type)
+                    segment_group = variable_pool.convert_template(parameter_value)
+                    parameter_value = segment_group.log if for_log else segment_group.text
+                    # variable_pool.convert_template returns a string,
+                    # so we need to convert it back to a dictionary
+                    try:
+                        if not isinstance(agent_input.value, str):
+                            parameter_value = json.loads(parameter_value)
+                    except json.JSONDecodeError:
+                        parameter_value = parameter_value
+                case _:
+                    raise AgentInputTypeError(agent_input.type)
             value = parameter_value
             if parameter.type == "array[tools]":
                 value = cast(list[dict[str, Any]], value)
@@ -374,12 +375,13 @@ class AgentNode(Node[AgentNodeData]):
         result: dict[str, Any] = {}
         for parameter_name in typed_node_data.agent_parameters:
             input = typed_node_data.agent_parameters[parameter_name]
-            if input.type in ["mixed", "constant"]:
-                selectors = VariableTemplateParser(str(input.value)).extract_variable_selectors()
-                for selector in selectors:
-                    result[selector.variable] = selector.value_selector
-            elif input.type == "variable":
-                result[parameter_name] = input.value
+            match input.type:
+                case "mixed" | "constant":
+                    selectors = VariableTemplateParser(str(input.value)).extract_variable_selectors()
+                    for selector in selectors:
+                        result[selector.variable] = selector.value_selector
+                case "variable":
+                    result[parameter_name] = input.value
 
         result = {node_id + "." + key: value for key, value in result.items()}
 

api/core/workflow/nodes/datasource/datasource_node.py

@@ -270,15 +270,18 @@ class DatasourceNode(Node[DatasourceNodeData]):
         if typed_node_data.datasource_parameters:
             for parameter_name in typed_node_data.datasource_parameters:
                 input = typed_node_data.datasource_parameters[parameter_name]
-                if input.type == "mixed":
-                    assert isinstance(input.value, str)
-                    selectors = VariableTemplateParser(input.value).extract_variable_selectors()
-                    for selector in selectors:
-                        result[selector.variable] = selector.value_selector
-                elif input.type == "variable":
-                    result[parameter_name] = input.value
-                elif input.type == "constant":
-                    pass
+                match input.type:
+                    case "mixed":
+                        assert isinstance(input.value, str)
+                        selectors = VariableTemplateParser(input.value).extract_variable_selectors()
+                        for selector in selectors:
+                            result[selector.variable] = selector.value_selector
+                    case "variable":
+                        result[parameter_name] = input.value
+                    case "constant":
+                        pass
+                    case None:
+                        pass
 
             result = {node_id + "." + key: value for key, value in result.items()}
 
@@ -308,99 +311,107 @@ class DatasourceNode(Node[DatasourceNodeData]):
         variables: dict[str, Any] = {}
 
         for message in message_stream:
-            if message.type in {
-                DatasourceMessage.MessageType.IMAGE_LINK,
-                DatasourceMessage.MessageType.BINARY_LINK,
-                DatasourceMessage.MessageType.IMAGE,
-            }:
-                assert isinstance(message.message, DatasourceMessage.TextMessage)
+            match message.type:
+                case (
+                    DatasourceMessage.MessageType.IMAGE_LINK
+                    | DatasourceMessage.MessageType.BINARY_LINK
+                    | DatasourceMessage.MessageType.IMAGE
+                ):
+                    assert isinstance(message.message, DatasourceMessage.TextMessage)
 
-                url = message.message.text
-                transfer_method = FileTransferMethod.TOOL_FILE
+                    url = message.message.text
+                    transfer_method = FileTransferMethod.TOOL_FILE
 
-                datasource_file_id = str(url).split("/")[-1].split(".")[0]
+                    datasource_file_id = str(url).split("/")[-1].split(".")[0]
 
-                with Session(db.engine) as session:
-                    stmt = select(ToolFile).where(ToolFile.id == datasource_file_id)
-                    datasource_file = session.scalar(stmt)
-                    if datasource_file is None:
-                        raise ToolFileError(f"Tool file {datasource_file_id} does not exist")
+                    with Session(db.engine) as session:
+                        stmt = select(ToolFile).where(ToolFile.id == datasource_file_id)
+                        datasource_file = session.scalar(stmt)
+                        if datasource_file is None:
+                            raise ToolFileError(f"Tool file {datasource_file_id} does not exist")
 
-                mapping = {
-                    "tool_file_id": datasource_file_id,
-                    "type": file_factory.get_file_type_by_mime_type(datasource_file.mimetype),
-                    "transfer_method": transfer_method,
-                    "url": url,
-                }
-                file = file_factory.build_from_mapping(
-                    mapping=mapping,
-                    tenant_id=self.tenant_id,
-                )
-                files.append(file)
-            elif message.type == DatasourceMessage.MessageType.BLOB:
-                # get tool file id
-                assert isinstance(message.message, DatasourceMessage.TextMessage)
-                assert message.meta
-
-                datasource_file_id = message.message.text.split("/")[-1].split(".")[0]
-                with Session(db.engine) as session:
-                    stmt = select(ToolFile).where(ToolFile.id == datasource_file_id)
-                    datasource_file = session.scalar(stmt)
-                    if datasource_file is None:
-                        raise ToolFileError(f"datasource file {datasource_file_id} not exists")
-
-                mapping = {
-                    "tool_file_id": datasource_file_id,
-                    "transfer_method": FileTransferMethod.TOOL_FILE,
-                }
-
-                files.append(
-                    file_factory.build_from_mapping(
+                    mapping = {
+                        "tool_file_id": datasource_file_id,
+                        "type": file_factory.get_file_type_by_mime_type(datasource_file.mimetype),
+                        "transfer_method": transfer_method,
+                        "url": url,
+                    }
+                    file = file_factory.build_from_mapping(
                         mapping=mapping,
                         tenant_id=self.tenant_id,
                     )
-                )
-            elif message.type == DatasourceMessage.MessageType.TEXT:
-                assert isinstance(message.message, DatasourceMessage.TextMessage)
-                text += message.message.text
-                yield StreamChunkEvent(
-                    selector=[self._node_id, "text"],
-                    chunk=message.message.text,
-                    is_final=False,
-                )
-            elif message.type == DatasourceMessage.MessageType.JSON:
-                assert isinstance(message.message, DatasourceMessage.JsonMessage)
-                json.append(message.message.json_object)
-            elif message.type == DatasourceMessage.MessageType.LINK:
-                assert isinstance(message.message, DatasourceMessage.TextMessage)
-                stream_text = f"Link: {message.message.text}\n"
-                text += stream_text
-                yield StreamChunkEvent(
-                    selector=[self._node_id, "text"],
-                    chunk=stream_text,
-                    is_final=False,
-                )
-            elif message.type == DatasourceMessage.MessageType.VARIABLE:
-                assert isinstance(message.message, DatasourceMessage.VariableMessage)
-                variable_name = message.message.variable_name
-                variable_value = message.message.variable_value
-                if message.message.stream:
-                    if not isinstance(variable_value, str):
-                        raise ValueError("When 'stream' is True, 'variable_value' must be a string.")
-                    if variable_name not in variables:
-                        variables[variable_name] = ""
-                    variables[variable_name] += variable_value
+                    files.append(file)
+                case DatasourceMessage.MessageType.BLOB:
+                    # get tool file id
+                    assert isinstance(message.message, DatasourceMessage.TextMessage)
+                    assert message.meta
 
+                    datasource_file_id = message.message.text.split("/")[-1].split(".")[0]
+                    with Session(db.engine) as session:
+                        stmt = select(ToolFile).where(ToolFile.id == datasource_file_id)
+                        datasource_file = session.scalar(stmt)
+                        if datasource_file is None:
+                            raise ToolFileError(f"datasource file {datasource_file_id} not exists")
+
+                    mapping = {
+                        "tool_file_id": datasource_file_id,
+                        "transfer_method": FileTransferMethod.TOOL_FILE,
+                    }
+
+                    files.append(
+                        file_factory.build_from_mapping(
+                            mapping=mapping,
+                            tenant_id=self.tenant_id,
+                        )
+                    )
+                case DatasourceMessage.MessageType.TEXT:
+                    assert isinstance(message.message, DatasourceMessage.TextMessage)
+                    text += message.message.text
                     yield StreamChunkEvent(
-                        selector=[self._node_id, variable_name],
-                        chunk=variable_value,
+                        selector=[self._node_id, "text"],
+                        chunk=message.message.text,
                         is_final=False,
                     )
-                else:
-                    variables[variable_name] = variable_value
-            elif message.type == DatasourceMessage.MessageType.FILE:
-                assert message.meta is not None
-                files.append(message.meta["file"])
+                case DatasourceMessage.MessageType.JSON:
+                    assert isinstance(message.message, DatasourceMessage.JsonMessage)
+                    json.append(message.message.json_object)
+                case DatasourceMessage.MessageType.LINK:
+                    assert isinstance(message.message, DatasourceMessage.TextMessage)
+                    stream_text = f"Link: {message.message.text}\n"
+                    text += stream_text
+                    yield StreamChunkEvent(
+                        selector=[self._node_id, "text"],
+                        chunk=stream_text,
+                        is_final=False,
+                    )
+                case DatasourceMessage.MessageType.VARIABLE:
+                    assert isinstance(message.message, DatasourceMessage.VariableMessage)
+                    variable_name = message.message.variable_name
+                    variable_value = message.message.variable_value
+                    if message.message.stream:
+                        if not isinstance(variable_value, str):
+                            raise ValueError("When 'stream' is True, 'variable_value' must be a string.")
+                        if variable_name not in variables:
+                            variables[variable_name] = ""
+                        variables[variable_name] += variable_value
+
+                        yield StreamChunkEvent(
+                            selector=[self._node_id, variable_name],
+                            chunk=variable_value,
+                            is_final=False,
+                        )
+                    else:
+                        variables[variable_name] = variable_value
+                case DatasourceMessage.MessageType.FILE:
+                    assert message.meta is not None
+                    files.append(message.meta["file"])
+                case (
+                    DatasourceMessage.MessageType.BLOB_CHUNK
+                    | DatasourceMessage.MessageType.LOG
+                    | DatasourceMessage.MessageType.RETRIEVER_RESOURCES
+                ):
+                    pass
+
         # mark the end of the stream
         yield StreamChunkEvent(
             selector=[self._node_id, "text"],

api/core/workflow/nodes/knowledge_index/knowledge_index_node.py

@@ -78,12 +78,21 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
                 indexing_technique = node_data.indexing_technique or dataset.indexing_technique
                 summary_index_setting = node_data.summary_index_setting or dataset.summary_index_setting
 
+                # Try to get document language if document_id is available
+                doc_language = None
+                document_id = variable_pool.get(["sys", SystemVariableKey.DOCUMENT_ID])
+                if document_id:
+                    document = db.session.query(Document).filter_by(id=document_id.value).first()
+                    if document and document.doc_language:
+                        doc_language = document.doc_language
+
                 outputs = self._get_preview_output_with_summaries(
                     node_data.chunk_structure,
                     chunks,
                     dataset=dataset,
                     indexing_technique=indexing_technique,
                     summary_index_setting=summary_index_setting,
+                    doc_language=doc_language,
                 )
                 return NodeRunResult(
                     status=WorkflowNodeExecutionStatus.SUCCEEDED,
@@ -315,6 +324,7 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
         dataset: Dataset,
         indexing_technique: str | None = None,
         summary_index_setting: dict | None = None,
+        doc_language: str | None = None,
     ) -> Mapping[str, Any]:
         """
         Generate preview output with summaries for chunks in preview mode.
@@ -326,6 +336,7 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
             dataset: Dataset object (for tenant_id)
             indexing_technique: Indexing technique from node config or dataset
             summary_index_setting: Summary index setting from node config or dataset
+            doc_language: Optional document language to ensure summary is generated in the correct language
         """
         index_processor = IndexProcessorFactory(chunk_structure).init_index_processor()
         preview_output = index_processor.format_preview(chunks)
@@ -365,6 +376,7 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
                                 tenant_id=dataset.tenant_id,
                                 text=preview_item["content"],
                                 summary_index_setting=summary_index_setting,
+                                document_language=doc_language,
                             )
                             if summary:
                                 preview_item["summary"] = summary
@@ -374,6 +386,7 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
                             tenant_id=dataset.tenant_id,
                             text=preview_item["content"],
                             summary_index_setting=summary_index_setting,
+                            document_language=doc_language,
                         )
                         if summary:
                             preview_item["summary"] = summary

api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py

@@ -303,33 +303,34 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
         elif str(node_data.retrieval_mode) == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE:
             if node_data.multiple_retrieval_config is None:
                 raise ValueError("multiple_retrieval_config is required")
-            if node_data.multiple_retrieval_config.reranking_mode == "reranking_model":
-                if node_data.multiple_retrieval_config.reranking_model:
-                    reranking_model = {
-                        "reranking_provider_name": node_data.multiple_retrieval_config.reranking_model.provider,
-                        "reranking_model_name": node_data.multiple_retrieval_config.reranking_model.model,
-                    }
-                else:
+            match node_data.multiple_retrieval_config.reranking_mode:
+                case "reranking_model":
+                    if node_data.multiple_retrieval_config.reranking_model:
+                        reranking_model = {
+                            "reranking_provider_name": node_data.multiple_retrieval_config.reranking_model.provider,
+                            "reranking_model_name": node_data.multiple_retrieval_config.reranking_model.model,
+                        }
+                    else:
+                        reranking_model = None
+                    weights = None
+                case "weighted_score":
+                    if node_data.multiple_retrieval_config.weights is None:
+                        raise ValueError("weights is required")
                     reranking_model = None
-                weights = None
-            elif node_data.multiple_retrieval_config.reranking_mode == "weighted_score":
-                if node_data.multiple_retrieval_config.weights is None:
-                    raise ValueError("weights is required")
-                reranking_model = None
-                vector_setting = node_data.multiple_retrieval_config.weights.vector_setting
-                weights = {
-                    "vector_setting": {
-                        "vector_weight": vector_setting.vector_weight,
-                        "embedding_provider_name": vector_setting.embedding_provider_name,
-                        "embedding_model_name": vector_setting.embedding_model_name,
-                    },
-                    "keyword_setting": {
-                        "keyword_weight": node_data.multiple_retrieval_config.weights.keyword_setting.keyword_weight
-                    },
-                }
-            else:
-                reranking_model = None
-                weights = None
+                    vector_setting = node_data.multiple_retrieval_config.weights.vector_setting
+                    weights = {
+                        "vector_setting": {
+                            "vector_weight": vector_setting.vector_weight,
+                            "embedding_provider_name": vector_setting.embedding_provider_name,
+                            "embedding_model_name": vector_setting.embedding_model_name,
+                        },
+                        "keyword_setting": {
+                            "keyword_weight": node_data.multiple_retrieval_config.weights.keyword_setting.keyword_weight
+                        },
+                    }
+                case _:
+                    reranking_model = None
+                    weights = None
             all_documents = dataset_retrieval.multiple_retrieve(
                 app_id=self.app_id,
                 tenant_id=self.tenant_id,
@@ -453,73 +454,74 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
         )
         filters: list[Any] = []
         metadata_condition = None
-        if node_data.metadata_filtering_mode == "disabled":
-            return None, None, usage
-        elif node_data.metadata_filtering_mode == "automatic":
-            automatic_metadata_filters, automatic_usage = self._automatic_metadata_filter_func(
-                dataset_ids, query, node_data
-            )
-            usage = self._merge_usage(usage, automatic_usage)
-            if automatic_metadata_filters:
-                conditions = []
-                for sequence, filter in enumerate(automatic_metadata_filters):
-                    DatasetRetrieval.process_metadata_filter_func(
-                        sequence,
-                        filter.get("condition", ""),
-                        filter.get("metadata_name", ""),
-                        filter.get("value"),
-                        filters,
-                    )
-                    conditions.append(
-                        Condition(
-                            name=filter.get("metadata_name"),  # type: ignore
-                            comparison_operator=filter.get("condition"),  # type: ignore
-                            value=filter.get("value"),
-                        )
-                    )
-                metadata_condition = MetadataCondition(
-                    logical_operator=node_data.metadata_filtering_conditions.logical_operator
-                    if node_data.metadata_filtering_conditions
-                    else "or",
-                    conditions=conditions,
+        match node_data.metadata_filtering_mode:
+            case "disabled":
+                return None, None, usage
+            case "automatic":
+                automatic_metadata_filters, automatic_usage = self._automatic_metadata_filter_func(
+                    dataset_ids, query, node_data
                 )
-        elif node_data.metadata_filtering_mode == "manual":
-            if node_data.metadata_filtering_conditions:
-                conditions = []
-                for sequence, condition in enumerate(node_data.metadata_filtering_conditions.conditions):  # type: ignore
-                    metadata_name = condition.name
-                    expected_value = condition.value
-                    if expected_value is not None and condition.comparison_operator not in ("empty", "not empty"):
-                        if isinstance(expected_value, str):
-                            expected_value = self.graph_runtime_state.variable_pool.convert_template(
-                                expected_value
-                            ).value[0]
-                            if expected_value.value_type in {"number", "integer", "float"}:
-                                expected_value = expected_value.value
-                            elif expected_value.value_type == "string":
-                                expected_value = re.sub(r"[\r\n\t]+", " ", expected_value.text).strip()
-                            else:
-                                raise ValueError("Invalid expected metadata value type")
-                    conditions.append(
-                        Condition(
-                            name=metadata_name,
-                            comparison_operator=condition.comparison_operator,
-                            value=expected_value,
+                usage = self._merge_usage(usage, automatic_usage)
+                if automatic_metadata_filters:
+                    conditions = []
+                    for sequence, filter in enumerate(automatic_metadata_filters):
+                        DatasetRetrieval.process_metadata_filter_func(
+                            sequence,
+                            filter.get("condition", ""),
+                            filter.get("metadata_name", ""),
+                            filter.get("value"),
+                            filters,
                         )
+                        conditions.append(
+                            Condition(
+                                name=filter.get("metadata_name"),  # type: ignore
+                                comparison_operator=filter.get("condition"),  # type: ignore
+                                value=filter.get("value"),
+                            )
+                        )
+                    metadata_condition = MetadataCondition(
+                        logical_operator=node_data.metadata_filtering_conditions.logical_operator
+                        if node_data.metadata_filtering_conditions
+                        else "or",
+                        conditions=conditions,
                     )
-                    filters = DatasetRetrieval.process_metadata_filter_func(
-                        sequence,
-                        condition.comparison_operator,
-                        metadata_name,
-                        expected_value,
-                        filters,
+            case "manual":
+                if node_data.metadata_filtering_conditions:
+                    conditions = []
+                    for sequence, condition in enumerate(node_data.metadata_filtering_conditions.conditions):  # type: ignore
+                        metadata_name = condition.name
+                        expected_value = condition.value
+                        if expected_value is not None and condition.comparison_operator not in ("empty", "not empty"):
+                            if isinstance(expected_value, str):
+                                expected_value = self.graph_runtime_state.variable_pool.convert_template(
+                                    expected_value
+                                ).value[0]
+                                if expected_value.value_type in {"number", "integer", "float"}:
+                                    expected_value = expected_value.value
+                                elif expected_value.value_type == "string":
+                                    expected_value = re.sub(r"[\r\n\t]+", " ", expected_value.text).strip()
+                                else:
+                                    raise ValueError("Invalid expected metadata value type")
+                        conditions.append(
+                            Condition(
+                                name=metadata_name,
+                                comparison_operator=condition.comparison_operator,
+                                value=expected_value,
+                            )
+                        )
+                        filters = DatasetRetrieval.process_metadata_filter_func(
+                            sequence,
+                            condition.comparison_operator,
+                            metadata_name,
+                            expected_value,
+                            filters,
+                        )
+                    metadata_condition = MetadataCondition(
+                        logical_operator=node_data.metadata_filtering_conditions.logical_operator,
+                        conditions=conditions,
                     )
-                metadata_condition = MetadataCondition(
-                    logical_operator=node_data.metadata_filtering_conditions.logical_operator,
-                    conditions=conditions,
-                )
-        else:
-            raise ValueError("Invalid metadata filtering mode")
+            case _:
+                raise ValueError("Invalid metadata filtering mode")
         if filters:
             if (
                 node_data.metadata_filtering_conditions

api/core/workflow/nodes/template_transform/template_transform_node.py

@@ -1,7 +1,6 @@
 from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any
 
-from configs import dify_config
 from core.workflow.enums import NodeType, WorkflowNodeExecutionStatus
 from core.workflow.node_events import NodeRunResult
 from core.workflow.nodes.base.node import Node
@@ -16,12 +15,13 @@ if TYPE_CHECKING:
     from core.workflow.entities import GraphInitParams
     from core.workflow.runtime import GraphRuntimeState
 
-MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH = dify_config.TEMPLATE_TRANSFORM_MAX_LENGTH
+DEFAULT_TEMPLATE_TRANSFORM_MAX_OUTPUT_LENGTH = 400_000
 
 
 class TemplateTransformNode(Node[TemplateTransformNodeData]):
     node_type = NodeType.TEMPLATE_TRANSFORM
     _template_renderer: Jinja2TemplateRenderer
+    _max_output_length: int
 
     def __init__(
         self,
@@ -31,6 +31,7 @@ class TemplateTransformNode(Node[TemplateTransformNodeData]):
         graph_runtime_state: "GraphRuntimeState",
         *,
         template_renderer: Jinja2TemplateRenderer | None = None,
+        max_output_length: int | None = None,
     ) -> None:
         super().__init__(
             id=id,
@@ -40,6 +41,10 @@ class TemplateTransformNode(Node[TemplateTransformNodeData]):
         )
         self._template_renderer = template_renderer or CodeExecutorJinja2TemplateRenderer()
 
+        if max_output_length is not None and max_output_length <= 0:
+            raise ValueError("max_output_length must be a positive integer")
+        self._max_output_length = max_output_length or DEFAULT_TEMPLATE_TRANSFORM_MAX_OUTPUT_LENGTH
+
     @classmethod
     def get_default_config(cls, filters: Mapping[str, object] | None = None) -> Mapping[str, object]:
         """
@@ -69,11 +74,11 @@ class TemplateTransformNode(Node[TemplateTransformNodeData]):
         except TemplateRenderError as e:
             return NodeRunResult(inputs=variables, status=WorkflowNodeExecutionStatus.FAILED, error=str(e))
 
-        if len(rendered) > MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH:
+        if len(rendered) > self._max_output_length:
             return NodeRunResult(
                 inputs=variables,
                 status=WorkflowNodeExecutionStatus.FAILED,
-                error=f"Output length exceeds {MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH} characters",
+                error=f"Output length exceeds {self._max_output_length} characters",
             )
 
         return NodeRunResult(

api/core/workflow/nodes/tool/tool_node.py

@@ -482,16 +482,17 @@ class ToolNode(Node[ToolNodeData]):
         result = {}
         for parameter_name in typed_node_data.tool_parameters:
             input = typed_node_data.tool_parameters[parameter_name]
-            if input.type == "mixed":
-                assert isinstance(input.value, str)
-                selectors = VariableTemplateParser(input.value).extract_variable_selectors()
-                for selector in selectors:
-                    result[selector.variable] = selector.value_selector
-            elif input.type == "variable":
-                selector_key = ".".join(input.value)
-                result[f"#{selector_key}#"] = input.value
-            elif input.type == "constant":
-                pass
+            match input.type:
+                case "mixed":
+                    assert isinstance(input.value, str)
+                    selectors = VariableTemplateParser(input.value).extract_variable_selectors()
+                    for selector in selectors:
+                        result[selector.variable] = selector.value_selector
+                case "variable":
+                    selector_key = ".".join(input.value)
+                    result[f"#{selector_key}#"] = input.value
+                case "constant":
+                    pass
 
         result = {node_id + "." + key: value for key, value in result.items()}
 

api/docker/entrypoint.sh

@@ -35,10 +35,10 @@ if [[ "${MODE}" == "worker" ]]; then
   if [[ -z "${CELERY_QUEUES}" ]]; then
     if [[ "${EDITION}" == "CLOUD" ]]; then
       # Cloud edition: separate queues for dataset and trigger tasks
-      DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
+      DEFAULT_QUEUES="api_token,dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow_professional,workflow_team,workflow_sandbox,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
     else
       # Community edition (SELF_HOSTED): dataset, pipeline and workflow have separate queues
-      DEFAULT_QUEUES="dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
+      DEFAULT_QUEUES="api_token,dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention"
     fi
   else
     DEFAULT_QUEUES="${CELERY_QUEUES}"

api/extensions/ext_celery.py

@@ -184,6 +184,14 @@ def init_app(app: DifyApp) -> Celery:
             "task": "schedule.trigger_provider_refresh_task.trigger_provider_refresh",
             "schedule": timedelta(minutes=dify_config.TRIGGER_PROVIDER_REFRESH_INTERVAL),
         }
+
+    if dify_config.ENABLE_API_TOKEN_LAST_USED_UPDATE_TASK:
+        imports.append("schedule.update_api_token_last_used_task")
+        beat_schedule["batch_update_api_token_last_used"] = {
+            "task": "schedule.update_api_token_last_used_task.batch_update_api_token_last_used",
+            "schedule": timedelta(minutes=dify_config.API_TOKEN_LAST_USED_UPDATE_INTERVAL),
+        }
+
     celery_app.conf.update(beat_schedule=beat_schedule, imports=imports)
 
     return celery_app

api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py

@@ -390,8 +390,7 @@ class ClickZettaVolumeStorage(BaseStorage):
         """
         content = self.load_once(filename)
 
-        with Path(target_filepath).open("wb") as f:
-            f.write(content)
+        Path(target_filepath).write_bytes(content)
 
         logger.debug("File %s downloaded from ClickZetta Volume to %s", filename, target_filepath)
 

api/fields/annotation_fields.py

@@ -1,36 +1,69 @@
-from flask_restx import Namespace, fields
+from __future__ import annotations
 
-from libs.helper import TimestampField
+from datetime import datetime
 
-annotation_fields = {
-    "id": fields.String,
-    "question": fields.String,
-    "answer": fields.Raw(attribute="content"),
-    "hit_count": fields.Integer,
-    "created_at": TimestampField,
-    # 'account': fields.Nested(simple_account_fields, allow_null=True)
-}
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
 
-def build_annotation_model(api_or_ns: Namespace):
-    """Build the annotation model for the API or Namespace."""
-    return api_or_ns.model("Annotation", annotation_fields)
+def _to_timestamp(value: datetime | int | None) -> int | None:
+    if isinstance(value, datetime):
+        return int(value.timestamp())
+    return value
 
 
-annotation_list_fields = {
-    "data": fields.List(fields.Nested(annotation_fields)),
-}
+class ResponseModel(BaseModel):
+    model_config = ConfigDict(
+        from_attributes=True,
+        extra="ignore",
+        populate_by_name=True,
+        serialize_by_alias=True,
+        protected_namespaces=(),
+    )
 
-annotation_hit_history_fields = {
-    "id": fields.String,
-    "source": fields.String,
-    "score": fields.Float,
-    "question": fields.String,
-    "created_at": TimestampField,
-    "match": fields.String(attribute="annotation_question"),
-    "response": fields.String(attribute="annotation_content"),
-}
 
-annotation_hit_history_list_fields = {
-    "data": fields.List(fields.Nested(annotation_hit_history_fields)),
-}
+class Annotation(ResponseModel):
+    id: str
+    question: str | None = None
+    answer: str | None = Field(default=None, validation_alias="content")
+    hit_count: int | None = None
+    created_at: int | None = None
+
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def _normalize_created_at(cls, value: datetime | int | None) -> int | None:
+        return _to_timestamp(value)
+
+
+class AnnotationList(ResponseModel):
+    data: list[Annotation]
+    has_more: bool
+    limit: int
+    total: int
+    page: int
+
+
+class AnnotationExportList(ResponseModel):
+    data: list[Annotation]
+
+
+class AnnotationHitHistory(ResponseModel):
+    id: str
+    source: str | None = None
+    score: float | None = None
+    question: str | None = None
+    created_at: int | None = None
+    match: str | None = Field(default=None, validation_alias="annotation_question")
+    response: str | None = Field(default=None, validation_alias="annotation_content")
+
+    @field_validator("created_at", mode="before")
+    @classmethod
+    def _normalize_created_at(cls, value: datetime | int | None) -> int | None:
+        return _to_timestamp(value)
+
+
+class AnnotationHitHistoryList(ResponseModel):
+    data: list[AnnotationHitHistory]
+    has_more: bool
+    limit: int
+    total: int
+    page: int

api/fields/end_user_fields.py

@@ -1,4 +1,7 @@
-from flask_restx import Namespace, fields
+from __future__ import annotations
+
+from flask_restx import fields
+from pydantic import BaseModel, ConfigDict
 
 simple_end_user_fields = {
     "id": fields.String,
@@ -8,5 +11,18 @@ simple_end_user_fields = {
 }
 
 
-def build_simple_end_user_model(api_or_ns: Namespace):
-    return api_or_ns.model("SimpleEndUser", simple_end_user_fields)
+class ResponseModel(BaseModel):
+    model_config = ConfigDict(
+        from_attributes=True,
+        extra="ignore",
+        populate_by_name=True,
+        serialize_by_alias=True,
+        protected_namespaces=(),
+    )
+
+
+class SimpleEndUser(ResponseModel):
+    id: str
+    type: str
+    is_anonymous: bool
+    session_id: str | None = None

api/fields/member_fields.py

@@ -1,6 +1,11 @@
-from flask_restx import Namespace, fields
+from __future__ import annotations
 
-from libs.helper import AvatarUrlField, TimestampField
+from datetime import datetime
+
+from flask_restx import fields
+from pydantic import BaseModel, ConfigDict, computed_field, field_validator
+
+from core.file import helpers as file_helpers
 
 simple_account_fields = {
     "id": fields.String,
@@ -9,36 +14,78 @@ simple_account_fields = {
 }
 
 
-def build_simple_account_model(api_or_ns: Namespace):
-    return api_or_ns.model("SimpleAccount", simple_account_fields)
+def _to_timestamp(value: datetime | int | None) -> int | None:
+    if isinstance(value, datetime):
+        return int(value.timestamp())
+    return value
 
 
-account_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "avatar": fields.String,
-    "avatar_url": AvatarUrlField,
-    "email": fields.String,
-    "is_password_set": fields.Boolean,
-    "interface_language": fields.String,
-    "interface_theme": fields.String,
-    "timezone": fields.String,
-    "last_login_at": TimestampField,
-    "last_login_ip": fields.String,
-    "created_at": TimestampField,
-}
+def _build_avatar_url(avatar: str | None) -> str | None:
+    if avatar is None:
+        return None
+    if avatar.startswith(("http://", "https://")):
+        return avatar
+    return file_helpers.get_signed_file_url(avatar)
 
-account_with_role_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "avatar": fields.String,
-    "avatar_url": AvatarUrlField,
-    "email": fields.String,
-    "last_login_at": TimestampField,
-    "last_active_at": TimestampField,
-    "created_at": TimestampField,
-    "role": fields.String,
-    "status": fields.String,
-}
 
-account_with_role_list_fields = {"accounts": fields.List(fields.Nested(account_with_role_fields))}
+class ResponseModel(BaseModel):
+    model_config = ConfigDict(
+        from_attributes=True,
+        extra="ignore",
+        populate_by_name=True,
+        serialize_by_alias=True,
+        protected_namespaces=(),
+    )
+
+
+class SimpleAccount(ResponseModel):
+    id: str
+    name: str
+    email: str
+
+
+class _AccountAvatar(ResponseModel):
+    avatar: str | None = None
+
+    @computed_field(return_type=str | None)  # type: ignore[prop-decorator]
+    @property
+    def avatar_url(self) -> str | None:
+        return _build_avatar_url(self.avatar)
+
+
+class Account(_AccountAvatar):
+    id: str
+    name: str
+    email: str
+    is_password_set: bool
+    interface_language: str | None = None
+    interface_theme: str | None = None
+    timezone: str | None = None
+    last_login_at: int | None = None
+    last_login_ip: str | None = None
+    created_at: int | None = None
+
+    @field_validator("last_login_at", "created_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
+        return _to_timestamp(value)
+
+
+class AccountWithRole(_AccountAvatar):
+    id: str
+    name: str
+    email: str
+    last_login_at: int | None = None
+    last_active_at: int | None = None
+    created_at: int | None = None
+    role: str
+    status: str
+
+    @field_validator("last_login_at", "last_active_at", "created_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
+        return _to_timestamp(value)
+
+
+class AccountWithRoleList(ResponseModel):
+    accounts: list[AccountWithRole]

api/fields/tag_fields.py

@@ -1,12 +1,20 @@
-from flask_restx import Namespace, fields
+from __future__ import annotations
 
-dataset_tag_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "type": fields.String,
-    "binding_count": fields.String,
-}
+from pydantic import BaseModel, ConfigDict
 
 
-def build_dataset_tag_fields(api_or_ns: Namespace):
-    return api_or_ns.model("DataSetTag", dataset_tag_fields)
+class ResponseModel(BaseModel):
+    model_config = ConfigDict(
+        from_attributes=True,
+        extra="ignore",
+        populate_by_name=True,
+        serialize_by_alias=True,
+        protected_namespaces=(),
+    )
+
+
+class DataSetTag(ResponseModel):
+    id: str
+    name: str
+    type: str
+    binding_count: str | None = None

api/fields/workflow_app_log_fields.py

@@ -1,7 +1,7 @@
 from flask_restx import Namespace, fields
 
-from fields.end_user_fields import build_simple_end_user_model, simple_end_user_fields
-from fields.member_fields import build_simple_account_model, simple_account_fields
+from fields.end_user_fields import simple_end_user_fields
+from fields.member_fields import simple_account_fields
 from fields.workflow_run_fields import (
     build_workflow_run_for_archived_log_model,
     build_workflow_run_for_log_model,
@@ -25,17 +25,9 @@ workflow_app_log_partial_fields = {
 def build_workflow_app_log_partial_model(api_or_ns: Namespace):
     """Build the workflow app log partial model for the API or Namespace."""
     workflow_run_model = build_workflow_run_for_log_model(api_or_ns)
-    simple_account_model = build_simple_account_model(api_or_ns)
-    simple_end_user_model = build_simple_end_user_model(api_or_ns)
 
     copied_fields = workflow_app_log_partial_fields.copy()
     copied_fields["workflow_run"] = fields.Nested(workflow_run_model, attribute="workflow_run", allow_null=True)
-    copied_fields["created_by_account"] = fields.Nested(
-        simple_account_model, attribute="created_by_account", allow_null=True
-    )
-    copied_fields["created_by_end_user"] = fields.Nested(
-        simple_end_user_model, attribute="created_by_end_user", allow_null=True
-    )
     return api_or_ns.model("WorkflowAppLogPartial", copied_fields)
 
 
@@ -52,17 +44,9 @@ workflow_archived_log_partial_fields = {
 def build_workflow_archived_log_partial_model(api_or_ns: Namespace):
     """Build the workflow archived log partial model for the API or Namespace."""
     workflow_run_model = build_workflow_run_for_archived_log_model(api_or_ns)
-    simple_account_model = build_simple_account_model(api_or_ns)
-    simple_end_user_model = build_simple_end_user_model(api_or_ns)
 
     copied_fields = workflow_archived_log_partial_fields.copy()
     copied_fields["workflow_run"] = fields.Nested(workflow_run_model, allow_null=True)
-    copied_fields["created_by_account"] = fields.Nested(
-        simple_account_model, attribute="created_by_account", allow_null=True
-    )
-    copied_fields["created_by_end_user"] = fields.Nested(
-        simple_end_user_model, attribute="created_by_end_user", allow_null=True
-    )
     return api_or_ns.model("WorkflowArchivedLogPartial", copied_fields)
 
 

api/migrations/versions/2025_12_25_1039-7df29de0f6be_add_credit_pool.py

@@ -10,6 +10,10 @@ import models as models
 import sqlalchemy as sa
 from sqlalchemy.dialects import postgresql
 
+
+def _is_pg(conn):
+    return conn.dialect.name == "postgresql"
+
 # revision identifiers, used by Alembic.
 revision = '7df29de0f6be'
 down_revision = '03ea244985ce'
@@ -19,16 +23,31 @@ depends_on = None
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('tenant_credit_pools',
-    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
-    sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
-    sa.Column('pool_type', sa.String(length=40), server_default='trial', nullable=False),
-    sa.Column('quota_limit', sa.BigInteger(), nullable=False),
-    sa.Column('quota_used', sa.BigInteger(), nullable=False),
-    sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
-    sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
-    sa.PrimaryKeyConstraint('id', name='tenant_credit_pool_pkey')
-    )
+    conn = op.get_bind()
+
+    if _is_pg(conn):
+        op.create_table('tenant_credit_pools',
+        sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
+        sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
+        sa.Column('pool_type', sa.String(length=40), server_default='trial', nullable=False),
+        sa.Column('quota_limit', sa.BigInteger(), nullable=False),
+        sa.Column('quota_used', sa.BigInteger(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), server_default=sa.text('CURRENT_TIMESTAMP'), nullable=False),
+        sa.PrimaryKeyConstraint('id', name='tenant_credit_pool_pkey')
+        )
+    else:
+        # For MySQL and other databases, UUID should be generated at application level
+        op.create_table('tenant_credit_pools',
+        sa.Column('id', models.types.StringUUID(), nullable=False),
+        sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
+        sa.Column('pool_type', sa.String(length=40), server_default='trial', nullable=False),
+        sa.Column('quota_limit', sa.BigInteger(), nullable=False),
+        sa.Column('quota_used', sa.BigInteger(), nullable=False),
+        sa.Column('created_at', sa.DateTime(), server_default=sa.func.current_timestamp(), nullable=False),
+        sa.Column('updated_at', sa.DateTime(), server_default=sa.func.current_timestamp(), nullable=False),
+        sa.PrimaryKeyConstraint('id', name='tenant_credit_pool_pkey')
+        )
     with op.batch_alter_table('tenant_credit_pools', schema=None) as batch_op:
         batch_op.create_index('tenant_credit_pool_pool_type_idx', ['pool_type'], unique=False)
         batch_op.create_index('tenant_credit_pool_tenant_id_idx', ['tenant_id'], unique=False)

api/models/model.py

@@ -2166,7 +2166,9 @@ class TenantCreditPool(TypeBase):
         sa.Index("tenant_credit_pool_pool_type_idx", "pool_type"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=text("uuid_generate_v4()"), init=False)
+    id: Mapped[str] = mapped_column(
+        StringUUID, insert_default=lambda: str(uuid4()), default_factory=lambda: str(uuid4()), init=False
+    )
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     pool_type: Mapped[str] = mapped_column(String(40), nullable=False, default="trial", server_default="trial")
     quota_limit: Mapped[int] = mapped_column(BigInteger, nullable=False, default=0)

api/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dify-api"
-version = "1.12.0"
+version = "1.12.1"
 requires-python = ">=3.11,<3.13"
 
 dependencies = [
@@ -145,7 +145,7 @@ dev = [
     "types-openpyxl~=3.1.5",
     "types-pexpect~=4.9.0",
     "types-protobuf~=5.29.1",
-    "types-psutil~=7.0.0",
+    "types-psutil~=7.2.2",
     "types-psycopg2~=2.9.21",
     "types-pygments~=2.19.0",
     "types-pymysql~=1.1.0",

api/schedule/check_upgradable_plugin_task.py

@@ -1,16 +1,24 @@
+import logging
 import math
 import time
 
 import click
 
 import app
+from core.helper.marketplace import fetch_global_plugin_manifest
 from extensions.ext_database import db
 from models.account import TenantPluginAutoUpgradeStrategy
 from tasks import process_tenant_plugin_autoupgrade_check_task as check_task
 
+logger = logging.getLogger(__name__)
+
 AUTO_UPGRADE_MINIMAL_CHECKING_INTERVAL = 15 * 60  # 15 minutes
 MAX_CONCURRENT_CHECK_TASKS = 20
 
+# Import cache constants from the task module
+CACHE_REDIS_KEY_PREFIX = check_task.CACHE_REDIS_KEY_PREFIX
+CACHE_REDIS_TTL = check_task.CACHE_REDIS_TTL
+
 
 @app.celery.task(queue="plugin")
 def check_upgradable_plugin_task():
@@ -40,6 +48,22 @@ def check_upgradable_plugin_task():
     )  # make sure all strategies are checked in this interval
     batch_interval_time = (AUTO_UPGRADE_MINIMAL_CHECKING_INTERVAL / batch_chunk_count) if batch_chunk_count > 0 else 0
 
+    if total_strategies == 0:
+        click.echo(click.style("no strategies to process, skipping plugin manifest fetch.", fg="green"))
+        return
+
+    # Fetch and cache all plugin manifests before processing tenants
+    # This reduces load on marketplace from 300k requests to 1 request per check cycle
+    logger.info("fetching global plugin manifest from marketplace")
+    try:
+        fetch_global_plugin_manifest(CACHE_REDIS_KEY_PREFIX, CACHE_REDIS_TTL)
+        logger.info("successfully fetched and cached global plugin manifest")
+    except Exception as e:
+        logger.exception("failed to fetch global plugin manifest")
+        click.echo(click.style(f"failed to fetch global plugin manifest: {e}", fg="red"))
+        click.echo(click.style("skipping plugin upgrade check for this cycle", fg="yellow"))
+        return
+
     for i in range(0, total_strategies, MAX_CONCURRENT_CHECK_TASKS):
         batch_strategies = strategies[i : i + MAX_CONCURRENT_CHECK_TASKS]
         for strategy in batch_strategies:

api/schedule/update_api_token_last_used_task.py

@@ -0,0 +1,114 @@
+"""
+Scheduled task to batch-update API token last_used_at timestamps.
+
+Instead of updating the database on every request, token usage is recorded
+in Redis as lightweight SET keys (api_token_active:{scope}:{token}).
+This task runs periodically (default every 30 minutes) to flush those
+records into the database in a single batch operation.
+"""
+
+import logging
+import time
+from datetime import datetime
+
+import click
+from sqlalchemy import update
+from sqlalchemy.orm import Session
+
+import app
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from models.model import ApiToken
+from services.api_token_service import ACTIVE_TOKEN_KEY_PREFIX
+
+logger = logging.getLogger(__name__)
+
+
+@app.celery.task(queue="api_token")
+def batch_update_api_token_last_used():
+    """
+    Batch update last_used_at for all recently active API tokens.
+
+    Scans Redis for api_token_active:* keys, parses the token and scope
+    from each key, and performs a batch database update.
+    """
+    click.echo(click.style("batch_update_api_token_last_used: start.", fg="green"))
+    start_at = time.perf_counter()
+
+    updated_count = 0
+    scanned_count = 0
+
+    try:
+        # Collect all active token keys and their values (the actual usage timestamps)
+        token_entries: list[tuple[str, str | None, datetime]] = []  # (token, scope, usage_time)
+        keys_to_delete: list[str | bytes] = []
+
+        for key in redis_client.scan_iter(match=f"{ACTIVE_TOKEN_KEY_PREFIX}*", count=200):
+            if isinstance(key, bytes):
+                key = key.decode("utf-8")
+            scanned_count += 1
+
+            # Read the value (ISO timestamp recorded at actual request time)
+            value = redis_client.get(key)
+            if not value:
+                keys_to_delete.append(key)
+                continue
+
+            if isinstance(value, bytes):
+                value = value.decode("utf-8")
+
+            try:
+                usage_time = datetime.fromisoformat(value)
+            except (ValueError, TypeError):
+                logger.warning("Invalid timestamp in key %s: %s", key, value)
+                keys_to_delete.append(key)
+                continue
+
+            # Parse token info from key: api_token_active:{scope}:{token}
+            suffix = key[len(ACTIVE_TOKEN_KEY_PREFIX) :]
+            parts = suffix.split(":", 1)
+            if len(parts) == 2:
+                scope_str, token = parts
+                scope = None if scope_str == "None" else scope_str
+                token_entries.append((token, scope, usage_time))
+            keys_to_delete.append(key)
+
+        if not token_entries:
+            click.echo(click.style("batch_update_api_token_last_used: no active tokens found.", fg="yellow"))
+            # Still clean up any invalid keys
+            if keys_to_delete:
+                redis_client.delete(*keys_to_delete)
+            return
+
+        # Update each token in its own short transaction to avoid long transactions
+        for token, scope, usage_time in token_entries:
+            with Session(db.engine, expire_on_commit=False) as session, session.begin():
+                stmt = (
+                    update(ApiToken)
+                    .where(
+                        ApiToken.token == token,
+                        ApiToken.type == scope,
+                        (ApiToken.last_used_at.is_(None) | (ApiToken.last_used_at < usage_time)),
+                    )
+                    .values(last_used_at=usage_time)
+                )
+                result = session.execute(stmt)
+                rowcount = getattr(result, "rowcount", 0)
+                if rowcount > 0:
+                    updated_count += 1
+
+        # Delete processed keys from Redis
+        if keys_to_delete:
+            redis_client.delete(*keys_to_delete)
+
+    except Exception:
+        logger.exception("batch_update_api_token_last_used failed")
+
+    elapsed = time.perf_counter() - start_at
+    click.echo(
+        click.style(
+            f"batch_update_api_token_last_used: done. "
+            f"scanned={scanned_count}, updated={updated_count}, elapsed={elapsed:.2f}s",
+            fg="green",
+        )
+    )

api/services/account_service.py

@@ -327,6 +327,17 @@ class AccountService:
     @staticmethod
     def delete_account(account: Account):
         """Delete account. This method only adds a task to the queue for deletion."""
+        # Queue account deletion sync tasks for all workspaces BEFORE account deletion (enterprise only)
+        from services.enterprise.account_deletion_sync import sync_account_deletion
+
+        sync_success = sync_account_deletion(account_id=account.id, source="account_deleted")
+        if not sync_success:
+            logger.warning(
+                "Enterprise account deletion sync failed for account %s; proceeding with local deletion.",
+                account.id,
+            )
+
+        # Now proceed with async account deletion
         delete_account_task.delay(account.id)
 
     @staticmethod
@@ -1230,6 +1241,19 @@ class TenantService:
         if dify_config.BILLING_ENABLED:
             BillingService.clean_billing_info_cache(tenant.id)
 
+        # Queue account deletion sync task for enterprise backend to reassign resources (enterprise only)
+        from services.enterprise.account_deletion_sync import sync_workspace_member_removal
+
+        sync_success = sync_workspace_member_removal(
+            workspace_id=tenant.id, member_id=account.id, source="workspace_member_removed"
+        )
+        if not sync_success:
+            logger.warning(
+                "Enterprise workspace member removal sync failed: workspace_id=%s, member_id=%s",
+                tenant.id,
+                account.id,
+            )
+
     @staticmethod
     def update_member_role(tenant: Tenant, member: Account, new_role: str, operator: Account):
         """Update member role"""

api/services/annotation_service.py

@@ -158,7 +158,7 @@ class AppAnnotationService:
                 .order_by(MessageAnnotation.created_at.desc(), MessageAnnotation.id.desc())
             )
         annotations = db.paginate(select=stmt, page=page, per_page=limit, max_per_page=100, error_out=False)
-        return annotations.items, annotations.total
+        return annotations.items, annotations.total or 0
 
     @classmethod
     def export_annotation_list_by_app_id(cls, app_id: str):
@@ -524,7 +524,7 @@ class AppAnnotationService:
         annotation_hit_histories = db.paginate(
             select=stmt, page=page, per_page=limit, max_per_page=100, error_out=False
         )
-        return annotation_hit_histories.items, annotation_hit_histories.total
+        return annotation_hit_histories.items, annotation_hit_histories.total or 0
 
     @classmethod
     def get_annotation_by_id(cls, annotation_id: str) -> MessageAnnotation | None:

api/services/api_token_service.py

@@ -0,0 +1,330 @@
+"""
+API Token Service
+
+Handles all API token caching, validation, and usage recording.
+Includes Redis cache operations, database queries, and single-flight concurrency control.
+"""
+
+import logging
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+from werkzeug.exceptions import Unauthorized
+
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client, redis_fallback
+from libs.datetime_utils import naive_utc_now
+from models.model import ApiToken
+
+logger = logging.getLogger(__name__)
+
+
+# ---------------------------------------------------------------------
+# Pydantic DTO
+# ---------------------------------------------------------------------
+
+
+class CachedApiToken(BaseModel):
+    """
+    Pydantic model for cached API token data.
+
+    This is NOT a SQLAlchemy model instance, but a plain Pydantic model
+    that mimics the ApiToken model interface for read-only access.
+    """
+
+    id: str
+    app_id: str | None
+    tenant_id: str | None
+    type: str
+    token: str
+    last_used_at: datetime | None
+    created_at: datetime | None
+
+    def __repr__(self) -> str:
+        return f"<CachedApiToken id={self.id} type={self.type}>"
+
+
+# ---------------------------------------------------------------------
+# Cache configuration
+# ---------------------------------------------------------------------
+
+CACHE_KEY_PREFIX = "api_token"
+CACHE_TTL_SECONDS = 600  # 10 minutes
+CACHE_NULL_TTL_SECONDS = 60  # 1 minute for non-existent tokens
+ACTIVE_TOKEN_KEY_PREFIX = "api_token_active:"
+
+
+# ---------------------------------------------------------------------
+# Cache class
+# ---------------------------------------------------------------------
+
+
+class ApiTokenCache:
+    """
+    Redis cache wrapper for API tokens.
+    Handles serialization, deserialization, and cache invalidation.
+    """
+
+    @staticmethod
+    def make_active_key(token: str, scope: str | None = None) -> str:
+        """Generate Redis key for recording token usage."""
+        return f"{ACTIVE_TOKEN_KEY_PREFIX}{scope}:{token}"
+
+    @staticmethod
+    def _make_tenant_index_key(tenant_id: str) -> str:
+        """Generate Redis key for tenant token index."""
+        return f"tenant_tokens:{tenant_id}"
+
+    @staticmethod
+    def _make_cache_key(token: str, scope: str | None = None) -> str:
+        """Generate cache key for the given token and scope."""
+        scope_str = scope or "any"
+        return f"{CACHE_KEY_PREFIX}:{scope_str}:{token}"
+
+    @staticmethod
+    def _serialize_token(api_token: Any) -> bytes:
+        """Serialize ApiToken object to JSON bytes."""
+        if isinstance(api_token, CachedApiToken):
+            return api_token.model_dump_json().encode("utf-8")
+
+        cached = CachedApiToken(
+            id=str(api_token.id),
+            app_id=str(api_token.app_id) if api_token.app_id else None,
+            tenant_id=str(api_token.tenant_id) if api_token.tenant_id else None,
+            type=api_token.type,
+            token=api_token.token,
+            last_used_at=api_token.last_used_at,
+            created_at=api_token.created_at,
+        )
+        return cached.model_dump_json().encode("utf-8")
+
+    @staticmethod
+    def _deserialize_token(cached_data: bytes | str) -> Any:
+        """Deserialize JSON bytes/string back to a CachedApiToken Pydantic model."""
+        if cached_data in {b"null", "null"}:
+            return None
+
+        try:
+            if isinstance(cached_data, bytes):
+                cached_data = cached_data.decode("utf-8")
+            return CachedApiToken.model_validate_json(cached_data)
+        except (ValueError, Exception) as e:
+            logger.warning("Failed to deserialize token from cache: %s", e)
+            return None
+
+    @staticmethod
+    @redis_fallback(default_return=None)
+    def get(token: str, scope: str | None) -> Any | None:
+        """Get API token from cache."""
+        cache_key = ApiTokenCache._make_cache_key(token, scope)
+        cached_data = redis_client.get(cache_key)
+
+        if cached_data is None:
+            logger.debug("Cache miss for token key: %s", cache_key)
+            return None
+
+        logger.debug("Cache hit for token key: %s", cache_key)
+        return ApiTokenCache._deserialize_token(cached_data)
+
+    @staticmethod
+    def _add_to_tenant_index(tenant_id: str | None, cache_key: str) -> None:
+        """Add cache key to tenant index for efficient invalidation."""
+        if not tenant_id:
+            return
+
+        try:
+            index_key = ApiTokenCache._make_tenant_index_key(tenant_id)
+            redis_client.sadd(index_key, cache_key)
+            redis_client.expire(index_key, CACHE_TTL_SECONDS + 60)
+        except Exception as e:
+            logger.warning("Failed to update tenant index: %s", e)
+
+    @staticmethod
+    def _remove_from_tenant_index(tenant_id: str | None, cache_key: str) -> None:
+        """Remove cache key from tenant index."""
+        if not tenant_id:
+            return
+
+        try:
+            index_key = ApiTokenCache._make_tenant_index_key(tenant_id)
+            redis_client.srem(index_key, cache_key)
+        except Exception as e:
+            logger.warning("Failed to remove from tenant index: %s", e)
+
+    @staticmethod
+    @redis_fallback(default_return=False)
+    def set(token: str, scope: str | None, api_token: Any | None, ttl: int = CACHE_TTL_SECONDS) -> bool:
+        """Set API token in cache."""
+        cache_key = ApiTokenCache._make_cache_key(token, scope)
+
+        if api_token is None:
+            cached_value = b"null"
+            ttl = CACHE_NULL_TTL_SECONDS
+        else:
+            cached_value = ApiTokenCache._serialize_token(api_token)
+
+        try:
+            redis_client.setex(cache_key, ttl, cached_value)
+
+            if api_token is not None and hasattr(api_token, "tenant_id"):
+                ApiTokenCache._add_to_tenant_index(api_token.tenant_id, cache_key)
+
+            logger.debug("Cached token with key: %s, ttl: %ss", cache_key, ttl)
+            return True
+        except Exception as e:
+            logger.warning("Failed to cache token: %s", e)
+            return False
+
+    @staticmethod
+    @redis_fallback(default_return=False)
+    def delete(token: str, scope: str | None = None) -> bool:
+        """Delete API token from cache."""
+        if scope is None:
+            pattern = f"{CACHE_KEY_PREFIX}:*:{token}"
+            try:
+                keys_to_delete = list(redis_client.scan_iter(match=pattern))
+                if keys_to_delete:
+                    redis_client.delete(*keys_to_delete)
+                    logger.info("Deleted %d cache entries for token", len(keys_to_delete))
+                return True
+            except Exception as e:
+                logger.warning("Failed to delete token cache with pattern: %s", e)
+                return False
+        else:
+            cache_key = ApiTokenCache._make_cache_key(token, scope)
+            try:
+                tenant_id = None
+                try:
+                    cached_data = redis_client.get(cache_key)
+                    if cached_data and cached_data != b"null":
+                        cached_token = ApiTokenCache._deserialize_token(cached_data)
+                        if cached_token:
+                            tenant_id = cached_token.tenant_id
+                except Exception as e:
+                    logger.debug("Failed to get tenant_id for cache cleanup: %s", e)
+
+                redis_client.delete(cache_key)
+
+                if tenant_id:
+                    ApiTokenCache._remove_from_tenant_index(tenant_id, cache_key)
+
+                logger.info("Deleted cache for key: %s", cache_key)
+                return True
+            except Exception as e:
+                logger.warning("Failed to delete token cache: %s", e)
+                return False
+
+    @staticmethod
+    @redis_fallback(default_return=False)
+    def invalidate_by_tenant(tenant_id: str) -> bool:
+        """Invalidate all API token caches for a specific tenant via tenant index."""
+        try:
+            index_key = ApiTokenCache._make_tenant_index_key(tenant_id)
+            cache_keys = redis_client.smembers(index_key)
+
+            if cache_keys:
+                deleted_count = 0
+                for cache_key in cache_keys:
+                    if isinstance(cache_key, bytes):
+                        cache_key = cache_key.decode("utf-8")
+                    redis_client.delete(cache_key)
+                    deleted_count += 1
+
+                redis_client.delete(index_key)
+
+                logger.info(
+                    "Invalidated %d token cache entries for tenant: %s",
+                    deleted_count,
+                    tenant_id,
+                )
+            else:
+                logger.info(
+                    "No tenant index found for %s, relying on TTL expiration",
+                    tenant_id,
+                )
+
+            return True
+
+        except Exception as e:
+            logger.warning("Failed to invalidate tenant token cache: %s", e)
+            return False
+
+
+# ---------------------------------------------------------------------
+# Token usage recording (for batch update)
+# ---------------------------------------------------------------------
+
+
+def record_token_usage(auth_token: str, scope: str | None) -> None:
+    """
+    Record token usage in Redis for later batch update by a scheduled job.
+
+    Instead of dispatching a Celery task per request, we simply SET a key in Redis.
+    A Celery Beat scheduled task will periodically scan these keys and batch-update
+    last_used_at in the database.
+    """
+    try:
+        key = ApiTokenCache.make_active_key(auth_token, scope)
+        redis_client.set(key, naive_utc_now().isoformat(), ex=3600)
+    except Exception as e:
+        logger.warning("Failed to record token usage: %s", e)
+
+
+# ---------------------------------------------------------------------
+# Database query + single-flight
+# ---------------------------------------------------------------------
+
+
+def query_token_from_db(auth_token: str, scope: str | None) -> ApiToken:
+    """
+    Query API token from database and cache the result.
+
+    Raises Unauthorized if token is invalid.
+    """
+    with Session(db.engine, expire_on_commit=False) as session:
+        stmt = select(ApiToken).where(ApiToken.token == auth_token, ApiToken.type == scope)
+        api_token = session.scalar(stmt)
+
+        if not api_token:
+            ApiTokenCache.set(auth_token, scope, None)
+            raise Unauthorized("Access token is invalid")
+
+        ApiTokenCache.set(auth_token, scope, api_token)
+        record_token_usage(auth_token, scope)
+        return api_token
+
+
+def fetch_token_with_single_flight(auth_token: str, scope: str | None) -> ApiToken | Any:
+    """
+    Fetch token from DB with single-flight pattern using Redis lock.
+
+    Ensures only one concurrent request queries the database for the same token.
+    Falls back to direct query if lock acquisition fails.
+    """
+    logger.debug("Token cache miss, attempting to acquire query lock for scope: %s", scope)
+
+    lock_key = f"api_token_query_lock:{scope}:{auth_token}"
+    lock = redis_client.lock(lock_key, timeout=10, blocking_timeout=5)
+
+    try:
+        if lock.acquire(blocking=True):
+            try:
+                cached_token = ApiTokenCache.get(auth_token, scope)
+                if cached_token is not None:
+                    logger.debug("Token cached by concurrent request, using cached version")
+                    return cached_token
+
+                return query_token_from_db(auth_token, scope)
+            finally:
+                lock.release()
+        else:
+            logger.warning("Lock timeout for token: %s, proceeding with direct query", auth_token[:10])
+            return query_token_from_db(auth_token, scope)
+    except Unauthorized:
+        raise
+    except Exception as e:
+        logger.warning("Redis lock failed for token query: %s, proceeding anyway", e)
+        return query_token_from_db(auth_token, scope)

api/services/dataset_service.py

@@ -16,6 +16,7 @@ from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden, NotFound
 
 from configs import dify_config
+from core.db.session_factory import session_factory
 from core.errors.error import LLMBadRequestError, ProviderTokenNotInitError
 from core.file import helpers as file_helpers
 from core.helper.name_generator import generate_incremental_name
@@ -1388,6 +1389,46 @@ class DocumentService:
         ).all()
         return documents
 
+    @staticmethod
+    def update_documents_need_summary(dataset_id: str, document_ids: Sequence[str], need_summary: bool = True) -> int:
+        """
+        Update need_summary field for multiple documents.
+
+        This method handles the case where documents were created when summary_index_setting was disabled,
+        and need to be updated when summary_index_setting is later enabled.
+
+        Args:
+            dataset_id: Dataset ID
+            document_ids: List of document IDs to update
+            need_summary: Value to set for need_summary field (default: True)
+
+        Returns:
+            Number of documents updated
+        """
+        if not document_ids:
+            return 0
+
+        document_id_list: list[str] = [str(document_id) for document_id in document_ids]
+
+        with session_factory.create_session() as session:
+            updated_count = (
+                session.query(Document)
+                .filter(
+                    Document.id.in_(document_id_list),
+                    Document.dataset_id == dataset_id,
+                    Document.doc_form != "qa_model",  # Skip qa_model documents
+                )
+                .update({Document.need_summary: need_summary}, synchronize_session=False)
+            )
+            session.commit()
+            logger.info(
+                "Updated need_summary to %s for %d documents in dataset %s",
+                need_summary,
+                updated_count,
+                dataset_id,
+            )
+            return updated_count
+
     @staticmethod
     def get_document_download_url(document: Document) -> str:
         """
@@ -2937,14 +2978,15 @@ class DocumentService:
         """
         now = naive_utc_now()
 
-        if action == "enable":
-            return DocumentService._prepare_enable_update(document, now)
-        elif action == "disable":
-            return DocumentService._prepare_disable_update(document, user, now)
-        elif action == "archive":
-            return DocumentService._prepare_archive_update(document, user, now)
-        elif action == "un_archive":
-            return DocumentService._prepare_unarchive_update(document, now)
+        match action:
+            case "enable":
+                return DocumentService._prepare_enable_update(document, now)
+            case "disable":
+                return DocumentService._prepare_disable_update(document, user, now)
+            case "archive":
+                return DocumentService._prepare_archive_update(document, user, now)
+            case "un_archive":
+                return DocumentService._prepare_unarchive_update(document, now)
 
         return None
 
@@ -3581,56 +3623,57 @@ class SegmentService:
         # Check if segment_ids is not empty to avoid WHERE false condition
         if not segment_ids or len(segment_ids) == 0:
             return
-        if action == "enable":
-            segments = db.session.scalars(
-                select(DocumentSegment).where(
-                    DocumentSegment.id.in_(segment_ids),
-                    DocumentSegment.dataset_id == dataset.id,
-                    DocumentSegment.document_id == document.id,
-                    DocumentSegment.enabled == False,
-                )
-            ).all()
-            if not segments:
-                return
-            real_deal_segment_ids = []
-            for segment in segments:
-                indexing_cache_key = f"segment_{segment.id}_indexing"
-                cache_result = redis_client.get(indexing_cache_key)
-                if cache_result is not None:
-                    continue
-                segment.enabled = True
-                segment.disabled_at = None
-                segment.disabled_by = None
-                db.session.add(segment)
-                real_deal_segment_ids.append(segment.id)
-            db.session.commit()
+        match action:
+            case "enable":
+                segments = db.session.scalars(
+                    select(DocumentSegment).where(
+                        DocumentSegment.id.in_(segment_ids),
+                        DocumentSegment.dataset_id == dataset.id,
+                        DocumentSegment.document_id == document.id,
+                        DocumentSegment.enabled == False,
+                    )
+                ).all()
+                if not segments:
+                    return
+                real_deal_segment_ids = []
+                for segment in segments:
+                    indexing_cache_key = f"segment_{segment.id}_indexing"
+                    cache_result = redis_client.get(indexing_cache_key)
+                    if cache_result is not None:
+                        continue
+                    segment.enabled = True
+                    segment.disabled_at = None
+                    segment.disabled_by = None
+                    db.session.add(segment)
+                    real_deal_segment_ids.append(segment.id)
+                db.session.commit()
 
-            enable_segments_to_index_task.delay(real_deal_segment_ids, dataset.id, document.id)
-        elif action == "disable":
-            segments = db.session.scalars(
-                select(DocumentSegment).where(
-                    DocumentSegment.id.in_(segment_ids),
-                    DocumentSegment.dataset_id == dataset.id,
-                    DocumentSegment.document_id == document.id,
-                    DocumentSegment.enabled == True,
-                )
-            ).all()
-            if not segments:
-                return
-            real_deal_segment_ids = []
-            for segment in segments:
-                indexing_cache_key = f"segment_{segment.id}_indexing"
-                cache_result = redis_client.get(indexing_cache_key)
-                if cache_result is not None:
-                    continue
-                segment.enabled = False
-                segment.disabled_at = naive_utc_now()
-                segment.disabled_by = current_user.id
-                db.session.add(segment)
-                real_deal_segment_ids.append(segment.id)
-            db.session.commit()
+                enable_segments_to_index_task.delay(real_deal_segment_ids, dataset.id, document.id)
+            case "disable":
+                segments = db.session.scalars(
+                    select(DocumentSegment).where(
+                        DocumentSegment.id.in_(segment_ids),
+                        DocumentSegment.dataset_id == dataset.id,
+                        DocumentSegment.document_id == document.id,
+                        DocumentSegment.enabled == True,
+                    )
+                ).all()
+                if not segments:
+                    return
+                real_deal_segment_ids = []
+                for segment in segments:
+                    indexing_cache_key = f"segment_{segment.id}_indexing"
+                    cache_result = redis_client.get(indexing_cache_key)
+                    if cache_result is not None:
+                        continue
+                    segment.enabled = False
+                    segment.disabled_at = naive_utc_now()
+                    segment.disabled_by = current_user.id
+                    db.session.add(segment)
+                    real_deal_segment_ids.append(segment.id)
+                db.session.commit()
 
-            disable_segments_from_index_task.delay(real_deal_segment_ids, dataset.id, document.id)
+                disable_segments_from_index_task.delay(real_deal_segment_ids, dataset.id, document.id)
 
     @classmethod
     def create_child_chunk(

api/services/enterprise/account_deletion_sync.py

@@ -0,0 +1,115 @@
+import json
+import logging
+import uuid
+from datetime import UTC, datetime
+
+from redis import RedisError
+
+from configs import dify_config
+from extensions.ext_database import db
+from extensions.ext_redis import redis_client
+from models.account import TenantAccountJoin
+
+logger = logging.getLogger(__name__)
+
+ACCOUNT_DELETION_SYNC_QUEUE = "enterprise:member:sync:queue"
+ACCOUNT_DELETION_SYNC_TASK_TYPE = "sync_member_deletion_from_workspace"
+
+
+def _queue_task(workspace_id: str, member_id: str, *, source: str) -> bool:
+    """
+    Queue an account deletion sync task to Redis.
+
+    Internal helper function. Do not call directly - use the public functions instead.
+
+    Args:
+        workspace_id: The workspace/tenant ID to sync
+        member_id: The member/account ID that was removed
+        source: Source of the sync request (for debugging/tracking)
+
+    Returns:
+        bool: True if task was queued successfully, False otherwise
+    """
+    try:
+        task = {
+            "task_id": str(uuid.uuid4()),
+            "workspace_id": workspace_id,
+            "member_id": member_id,
+            "retry_count": 0,
+            "created_at": datetime.now(UTC).isoformat(),
+            "source": source,
+            "type": ACCOUNT_DELETION_SYNC_TASK_TYPE,
+        }
+
+        # Push to Redis list (queue) - LPUSH adds to the head, worker consumes from tail with RPOP
+        redis_client.lpush(ACCOUNT_DELETION_SYNC_QUEUE, json.dumps(task))
+
+        logger.info(
+            "Queued account deletion sync task for workspace %s, member %s, task_id: %s, source: %s",
+            workspace_id,
+            member_id,
+            task["task_id"],
+            source,
+        )
+        return True
+
+    except (RedisError, TypeError) as e:
+        logger.error(
+            "Failed to queue account deletion sync for workspace %s, member %s: %s",
+            workspace_id,
+            member_id,
+            str(e),
+            exc_info=True,
+        )
+        # Don't raise - we don't want to fail member deletion if queueing fails
+        return False
+
+
+def sync_workspace_member_removal(workspace_id: str, member_id: str, *, source: str) -> bool:
+    """
+    Sync a single workspace member removal (enterprise only).
+
+    Queues a task for the enterprise backend to reassign resources from the removed member.
+    Handles enterprise edition check internally. Safe to call in community edition (no-op).
+
+    Args:
+        workspace_id: The workspace/tenant ID
+        member_id: The member/account ID that was removed
+        source: Source of the sync request (e.g., "workspace_member_removed")
+
+    Returns:
+        bool: True if task was queued (or skipped in community), False if queueing failed
+    """
+    if not dify_config.ENTERPRISE_ENABLED:
+        return True
+
+    return _queue_task(workspace_id=workspace_id, member_id=member_id, source=source)
+
+
+def sync_account_deletion(account_id: str, *, source: str) -> bool:
+    """
+    Sync full account deletion across all workspaces (enterprise only).
+
+    Fetches all workspace memberships for the account and queues a sync task for each.
+    Handles enterprise edition check internally. Safe to call in community edition (no-op).
+
+    Args:
+        account_id: The account ID being deleted
+        source: Source of the sync request (e.g., "account_deleted")
+
+    Returns:
+        bool: True if all tasks were queued (or skipped in community), False if any queueing failed
+    """
+    if not dify_config.ENTERPRISE_ENABLED:
+        return True
+
+    # Fetch all workspaces the account belongs to
+    workspace_joins = db.session.query(TenantAccountJoin).filter_by(account_id=account_id).all()
+
+    # Queue sync task for each workspace
+    success = True
+    for join in workspace_joins:
+        if not _queue_task(workspace_id=join.tenant_id, member_id=account_id, source=source):
+            success = False
+
+    return success

api/services/rag_pipeline/rag_pipeline_transform_service.py

@@ -174,6 +174,10 @@ class RagPipelineTransformService:
         else:
             dataset.retrieval_model = knowledge_configuration.retrieval_model.model_dump()
 
+        # Copy summary_index_setting from dataset to knowledge_index node configuration
+        if dataset.summary_index_setting:
+            knowledge_configuration.summary_index_setting = dataset.summary_index_setting
+
         knowledge_configuration_dict.update(knowledge_configuration.model_dump())
         node["data"] = knowledge_configuration_dict
         return node

api/services/summary_index_service.py

@@ -49,11 +49,18 @@ class SummaryIndexService:
         # Use lazy import to avoid circular import
         from core.rag.index_processor.processor.paragraph_index_processor import ParagraphIndexProcessor
 
+        # Get document language to ensure summary is generated in the correct language
+        # This is especially important for image-only chunks where text is empty or minimal
+        document_language = None
+        if segment.document and segment.document.doc_language:
+            document_language = segment.document.doc_language
+
         summary_content, usage = ParagraphIndexProcessor.generate_summary(
             tenant_id=dataset.tenant_id,
             text=segment.content,
             summary_index_setting=summary_index_setting,
             segment_id=segment.id,
+            document_language=document_language,
         )
 
         if not summary_content:
@@ -558,6 +565,9 @@ class SummaryIndexService:
                     )
                     session.add(summary_record)
 
+            # Commit the batch created records
+            session.commit()
+
     @staticmethod
     def update_summary_record_error(
         segment: DocumentSegment,
@@ -762,7 +772,6 @@ class SummaryIndexService:
                 dataset=dataset,
                 status="not_started",
             )
-            session.commit()  # Commit initial records
 
             summary_records = []
 

api/services/tag_service.py

@@ -24,7 +24,7 @@ class TagService:
             escaped_keyword = escape_like_pattern(keyword)
             query = query.where(sa.and_(Tag.name.ilike(f"%{escaped_keyword}%", escape="\\")))
         query = query.group_by(Tag.id, Tag.type, Tag.name, Tag.created_at)
-        results = query.order_by(Tag.created_at.desc()).all()
+        results: list = query.order_by(Tag.created_at.desc()).all()
         return results
 
     @staticmethod

api/tasks/batch_clean_document_task.py

@@ -14,6 +14,9 @@ from models.model import UploadFile
 
 logger = logging.getLogger(__name__)
 
+# Batch size for database operations to keep transactions short
+BATCH_SIZE = 1000
+
 
 @shared_task(queue="dataset")
 def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form: str | None, file_ids: list[str]):
@@ -31,63 +34,179 @@ def batch_clean_document_task(document_ids: list[str], dataset_id: str, doc_form
     if not doc_form:
         raise ValueError("doc_form is required")
 
-    with session_factory.create_session() as session:
-        try:
-            dataset = session.query(Dataset).where(Dataset.id == dataset_id).first()
-
-            if not dataset:
-                raise Exception("Document has no dataset")
-
-            session.query(DatasetMetadataBinding).where(
-                DatasetMetadataBinding.dataset_id == dataset_id,
-                DatasetMetadataBinding.document_id.in_(document_ids),
-            ).delete(synchronize_session=False)
+    storage_keys_to_delete: list[str] = []
+    index_node_ids: list[str] = []
+    segment_ids: list[str] = []
+    total_image_upload_file_ids: list[str] = []
 
+    try:
+        # ============ Step 1: Query segment and file data (short read-only transaction) ============
+        with session_factory.create_session() as session:
+            # Get segments info
             segments = session.scalars(
                 select(DocumentSegment).where(DocumentSegment.document_id.in_(document_ids))
             ).all()
-            # check segment is exist
+
             if segments:
                 index_node_ids = [segment.index_node_id for segment in segments]
-                index_processor = IndexProcessorFactory(doc_form).init_index_processor()
-                index_processor.clean(
-                    dataset, index_node_ids, with_keywords=True, delete_child_chunks=True, delete_summaries=True
-                )
+                segment_ids = [segment.id for segment in segments]
 
+                # Collect image file IDs from segment content
                 for segment in segments:
                     image_upload_file_ids = get_image_upload_file_ids(segment.content)
-                    image_files = session.query(UploadFile).where(UploadFile.id.in_(image_upload_file_ids)).all()
-                    for image_file in image_files:
-                        try:
-                            if image_file and image_file.key:
-                                storage.delete(image_file.key)
-                        except Exception:
-                            logger.exception(
-                                "Delete image_files failed when storage deleted, \
-                                              image_upload_file_is: %s",
-                                image_file.id,
-                            )
-                    stmt = delete(UploadFile).where(UploadFile.id.in_(image_upload_file_ids))
-                    session.execute(stmt)
-                    session.delete(segment)
+                    total_image_upload_file_ids.extend(image_upload_file_ids)
+
+            # Query storage keys for image files
+            if total_image_upload_file_ids:
+                image_files = session.scalars(
+                    select(UploadFile).where(UploadFile.id.in_(total_image_upload_file_ids))
+                ).all()
+                storage_keys_to_delete.extend([f.key for f in image_files if f and f.key])
+
+            # Query storage keys for document files
             if file_ids:
                 files = session.scalars(select(UploadFile).where(UploadFile.id.in_(file_ids))).all()
-                for file in files:
-                    try:
-                        storage.delete(file.key)
-                    except Exception:
-                        logger.exception("Delete file failed when document deleted, file_id: %s", file.id)
-                stmt = delete(UploadFile).where(UploadFile.id.in_(file_ids))
-                session.execute(stmt)
+                storage_keys_to_delete.extend([f.key for f in files if f and f.key])
 
-            session.commit()
-
-            end_at = time.perf_counter()
-            logger.info(
-                click.style(
-                    f"Cleaned documents when documents deleted latency: {end_at - start_at}",
-                    fg="green",
+        # ============ Step 2: Clean vector index (external service, fresh session for dataset) ============
+        if index_node_ids:
+            try:
+                # Fetch dataset in a fresh session to avoid DetachedInstanceError
+                with session_factory.create_session() as session:
+                    dataset = session.query(Dataset).where(Dataset.id == dataset_id).first()
+                    if not dataset:
+                        logger.warning("Dataset not found for vector index cleanup, dataset_id: %s", dataset_id)
+                    else:
+                        index_processor = IndexProcessorFactory(doc_form).init_index_processor()
+                        index_processor.clean(
+                            dataset, index_node_ids, with_keywords=True, delete_child_chunks=True, delete_summaries=True
+                        )
+            except Exception:
+                logger.exception(
+                    "Failed to clean vector index for dataset_id: %s, document_ids: %s, index_node_ids count: %d",
+                    dataset_id,
+                    document_ids,
+                    len(index_node_ids),
                 )
-            )
+
+        # ============ Step 3: Delete metadata binding (separate short transaction) ============
+        try:
+            with session_factory.create_session() as session:
+                deleted_count = (
+                    session.query(DatasetMetadataBinding)
+                    .where(
+                        DatasetMetadataBinding.dataset_id == dataset_id,
+                        DatasetMetadataBinding.document_id.in_(document_ids),
+                    )
+                    .delete(synchronize_session=False)
+                )
+                session.commit()
+                logger.debug("Deleted %d metadata bindings for dataset_id: %s", deleted_count, dataset_id)
         except Exception:
-            logger.exception("Cleaned documents when documents deleted failed")
+            logger.exception(
+                "Failed to delete metadata bindings for dataset_id: %s, document_ids: %s",
+                dataset_id,
+                document_ids,
+            )
+
+        # ============ Step 4: Batch delete UploadFile records (multiple short transactions) ============
+        if total_image_upload_file_ids:
+            failed_batches = 0
+            total_batches = (len(total_image_upload_file_ids) + BATCH_SIZE - 1) // BATCH_SIZE
+            for i in range(0, len(total_image_upload_file_ids), BATCH_SIZE):
+                batch = total_image_upload_file_ids[i : i + BATCH_SIZE]
+                try:
+                    with session_factory.create_session() as session:
+                        stmt = delete(UploadFile).where(UploadFile.id.in_(batch))
+                        session.execute(stmt)
+                        session.commit()
+                except Exception:
+                    failed_batches += 1
+                    logger.exception(
+                        "Failed to delete image UploadFile batch %d-%d for dataset_id: %s",
+                        i,
+                        i + len(batch),
+                        dataset_id,
+                    )
+            if failed_batches > 0:
+                logger.warning(
+                    "Image UploadFile deletion: %d/%d batches failed for dataset_id: %s",
+                    failed_batches,
+                    total_batches,
+                    dataset_id,
+                )
+
+        # ============ Step 5: Batch delete DocumentSegment records (multiple short transactions) ============
+        if segment_ids:
+            failed_batches = 0
+            total_batches = (len(segment_ids) + BATCH_SIZE - 1) // BATCH_SIZE
+            for i in range(0, len(segment_ids), BATCH_SIZE):
+                batch = segment_ids[i : i + BATCH_SIZE]
+                try:
+                    with session_factory.create_session() as session:
+                        segment_delete_stmt = delete(DocumentSegment).where(DocumentSegment.id.in_(batch))
+                        session.execute(segment_delete_stmt)
+                        session.commit()
+                except Exception:
+                    failed_batches += 1
+                    logger.exception(
+                        "Failed to delete DocumentSegment batch %d-%d for dataset_id: %s, document_ids: %s",
+                        i,
+                        i + len(batch),
+                        dataset_id,
+                        document_ids,
+                    )
+            if failed_batches > 0:
+                logger.warning(
+                    "DocumentSegment deletion: %d/%d batches failed, document_ids: %s",
+                    failed_batches,
+                    total_batches,
+                    document_ids,
+                )
+
+        # ============ Step 6: Delete document-associated files (separate short transaction) ============
+        if file_ids:
+            try:
+                with session_factory.create_session() as session:
+                    stmt = delete(UploadFile).where(UploadFile.id.in_(file_ids))
+                    session.execute(stmt)
+                    session.commit()
+            except Exception:
+                logger.exception(
+                    "Failed to delete document UploadFile records for dataset_id: %s, file_ids: %s",
+                    dataset_id,
+                    file_ids,
+                )
+
+        # ============ Step 7: Delete storage files (I/O operations, no DB transaction) ============
+        storage_delete_failures = 0
+        for storage_key in storage_keys_to_delete:
+            try:
+                storage.delete(storage_key)
+            except Exception:
+                storage_delete_failures += 1
+                logger.exception("Failed to delete file from storage, key: %s", storage_key)
+        if storage_delete_failures > 0:
+            logger.warning(
+                "Storage file deletion completed with %d failures out of %d total files for dataset_id: %s",
+                storage_delete_failures,
+                len(storage_keys_to_delete),
+                dataset_id,
+            )
+
+        end_at = time.perf_counter()
+        logger.info(
+            click.style(
+                f"Cleaned documents when documents deleted latency: {end_at - start_at:.2f}s, "
+                f"dataset_id: {dataset_id}, document_ids: {document_ids}, "
+                f"segments: {len(segment_ids)}, image_files: {len(total_image_upload_file_ids)}, "
+                f"storage_files: {len(storage_keys_to_delete)}",
+                fg="green",
+            )
+        )
+    except Exception:
+        logger.exception(
+            "Batch clean documents failed for dataset_id: %s, document_ids: %s",
+            dataset_id,
+            document_ids,
+        )

api/tasks/delete_segment_from_index_task.py

@@ -3,6 +3,7 @@ import time
 
 import click
 from celery import shared_task
+from sqlalchemy import delete
 
 from core.db.session_factory import session_factory
 from core.rag.index_processor.index_processor_factory import IndexProcessorFactory
@@ -67,8 +68,14 @@ def delete_segment_from_index_task(
                 if segment_attachment_bindings:
                     attachment_ids = [binding.attachment_id for binding in segment_attachment_bindings]
                     index_processor.clean(dataset=dataset, node_ids=attachment_ids, with_keywords=False)
-                    for binding in segment_attachment_bindings:
-                        session.delete(binding)
+                    segment_attachment_bind_ids = [i.id for i in segment_attachment_bindings]
+
+                    for i in range(0, len(segment_attachment_bind_ids), 1000):
+                        segment_attachment_bind_delete_stmt = delete(SegmentAttachmentBinding).where(
+                            SegmentAttachmentBinding.id.in_(segment_attachment_bind_ids[i : i + 1000])
+                        )
+                        session.execute(segment_attachment_bind_delete_stmt)
+
                     # delete upload file
                     session.query(UploadFile).where(UploadFile.id.in_(attachment_ids)).delete(synchronize_session=False)
                     session.commit()

api/tasks/document_indexing_sync_task.py

@@ -28,7 +28,7 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
     logger.info(click.style(f"Start sync document: {document_id}", fg="green"))
     start_at = time.perf_counter()
 
-    with session_factory.create_session() as session:
+    with session_factory.create_session() as session, session.begin():
         document = session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
 
         if not document:
@@ -68,7 +68,6 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
                 document.indexing_status = "error"
                 document.error = "Datasource credential not found. Please reconnect your Notion workspace."
                 document.stopped_at = naive_utc_now()
-                session.commit()
                 return
 
             loader = NotionExtractor(
@@ -85,7 +84,6 @@ def document_indexing_sync_task(dataset_id: str, document_id: str):
             if last_edited_time != page_edited_time:
                 document.indexing_status = "parsing"
                 document.processing_started_at = naive_utc_now()
-                session.commit()
 
                 # delete all document segment and index
                 try:

api/tasks/document_indexing_update_task.py

@@ -8,7 +8,6 @@ from sqlalchemy import delete, select
 from core.db.session_factory import session_factory
 from core.indexing_runner import DocumentIsPausedError, IndexingRunner
 from core.rag.index_processor.index_processor_factory import IndexProcessorFactory
-from extensions.ext_database import db
 from libs.datetime_utils import naive_utc_now
 from models.dataset import Dataset, Document, DocumentSegment
 
@@ -27,7 +26,7 @@ def document_indexing_update_task(dataset_id: str, document_id: str):
     logger.info(click.style(f"Start update document: {document_id}", fg="green"))
     start_at = time.perf_counter()
 
-    with session_factory.create_session() as session:
+    with session_factory.create_session() as session, session.begin():
         document = session.query(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).first()
 
         if not document:
@@ -36,7 +35,6 @@ def document_indexing_update_task(dataset_id: str, document_id: str):
 
         document.indexing_status = "parsing"
         document.processing_started_at = naive_utc_now()
-        session.commit()
 
         # delete all document segment and index
         try:
@@ -56,7 +54,7 @@ def document_indexing_update_task(dataset_id: str, document_id: str):
                 segment_ids = [segment.id for segment in segments]
                 segment_delete_stmt = delete(DocumentSegment).where(DocumentSegment.id.in_(segment_ids))
                 session.execute(segment_delete_stmt)
-                db.session.commit()
+
             end_at = time.perf_counter()
             logger.info(
                 click.style(

api/tasks/process_tenant_plugin_autoupgrade_check_task.py

@@ -6,8 +6,8 @@ import typing
 import click
 from celery import shared_task
 
-from core.helper import marketplace
-from core.helper.marketplace import MarketplacePluginDeclaration
+from core.helper.marketplace import record_install_plugin_event
+from core.plugin.entities.marketplace import MarketplacePluginSnapshot
 from core.plugin.entities.plugin import PluginInstallationSource
 from core.plugin.impl.plugin import PluginInstaller
 from extensions.ext_redis import redis_client
@@ -16,7 +16,7 @@ from models.account import TenantPluginAutoUpgradeStrategy
 logger = logging.getLogger(__name__)
 
 RETRY_TIMES_OF_ONE_PLUGIN_IN_ONE_TENANT = 3
-CACHE_REDIS_KEY_PREFIX = "plugin_autoupgrade_check_task:cached_plugin_manifests:"
+CACHE_REDIS_KEY_PREFIX = "plugin_autoupgrade_check_task:cached_plugin_snapshot:"
 CACHE_REDIS_TTL = 60 * 60  # 1 hour
 
 
@@ -25,11 +25,11 @@ def _get_redis_cache_key(plugin_id: str) -> str:
     return f"{CACHE_REDIS_KEY_PREFIX}{plugin_id}"
 
 
-def _get_cached_manifest(plugin_id: str) -> typing.Union[MarketplacePluginDeclaration, None, bool]:
+def _get_cached_manifest(plugin_id: str) -> typing.Union[MarketplacePluginSnapshot, None, bool]:
     """
     Get cached plugin manifest from Redis.
     Returns:
-        - MarketplacePluginDeclaration: if found in cache
+        - MarketplacePluginSnapshot: if found in cache
         - None: if cached as not found (marketplace returned no result)
         - False: if not in cache at all
     """
@@ -43,76 +43,31 @@ def _get_cached_manifest(plugin_id: str) -> typing.Union[MarketplacePluginDeclar
         if cached_json is None:
             return None
 
-        return MarketplacePluginDeclaration.model_validate(cached_json)
+        return MarketplacePluginSnapshot.model_validate(cached_json)
     except Exception:
         logger.exception("Failed to get cached manifest for plugin %s", plugin_id)
         return False
 
 
-def _set_cached_manifest(plugin_id: str, manifest: typing.Union[MarketplacePluginDeclaration, None]) -> None:
-    """
-    Cache plugin manifest in Redis.
-    Args:
-        plugin_id: The plugin ID
-        manifest: The manifest to cache, or None if not found in marketplace
-    """
-    try:
-        key = _get_redis_cache_key(plugin_id)
-        if manifest is None:
-            # Cache the fact that this plugin was not found
-            redis_client.setex(key, CACHE_REDIS_TTL, json.dumps(None))
-        else:
-            # Cache the manifest data
-            redis_client.setex(key, CACHE_REDIS_TTL, manifest.model_dump_json())
-    except Exception:
-        # If Redis fails, continue without caching
-        # traceback.print_exc()
-        logger.exception("Failed to set cached manifest for plugin %s", plugin_id)
-
-
 def marketplace_batch_fetch_plugin_manifests(
     plugin_ids_plain_list: list[str],
-) -> list[MarketplacePluginDeclaration]:
-    """Fetch plugin manifests with Redis caching support."""
-    cached_manifests: dict[str, typing.Union[MarketplacePluginDeclaration, None]] = {}
-    not_cached_plugin_ids: list[str] = []
+) -> list[MarketplacePluginSnapshot]:
+    """
+    Fetch plugin manifests from Redis cache only.
+    This function assumes fetch_global_plugin_manifest() has been called
+    to pre-populate the cache with all marketplace plugins.
+    """
+    result: list[MarketplacePluginSnapshot] = []
 
     # Check Redis cache for each plugin
     for plugin_id in plugin_ids_plain_list:
         cached_result = _get_cached_manifest(plugin_id)
-        if cached_result is False:
-            # Not in cache, need to fetch
-            not_cached_plugin_ids.append(plugin_id)
-        else:
-            # Either found manifest or cached as None (not found in marketplace)
-            # At this point, cached_result is either MarketplacePluginDeclaration or None
-            if isinstance(cached_result, bool):
-                # This should never happen due to the if condition above, but for type safety
-                continue
-            cached_manifests[plugin_id] = cached_result
+        if not isinstance(cached_result, MarketplacePluginSnapshot):
+            # cached_result is False (not in cache) or None (cached as not found)
+            logger.warning("plugin %s not found in cache, skipping", plugin_id)
+            continue
 
-    # Fetch uncached plugins from marketplace
-    if not_cached_plugin_ids:
-        manifests = marketplace.batch_fetch_plugin_manifests_ignore_deserialization_error(not_cached_plugin_ids)
-
-        # Cache the fetched manifests
-        for manifest in manifests:
-            cached_manifests[manifest.plugin_id] = manifest
-            _set_cached_manifest(manifest.plugin_id, manifest)
-
-        # Cache plugins that were not found in marketplace
-        fetched_plugin_ids = {manifest.plugin_id for manifest in manifests}
-        for plugin_id in not_cached_plugin_ids:
-            if plugin_id not in fetched_plugin_ids:
-                cached_manifests[plugin_id] = None
-                _set_cached_manifest(plugin_id, None)
-
-    # Build result list from cached manifests
-    result: list[MarketplacePluginDeclaration] = []
-    for plugin_id in plugin_ids_plain_list:
-        cached_manifest: typing.Union[MarketplacePluginDeclaration, None] = cached_manifests.get(plugin_id)
-        if cached_manifest is not None:
-            result.append(cached_manifest)
+        result.append(cached_result)
 
     return result
 
@@ -211,7 +166,7 @@ def process_tenant_plugin_autoupgrade_check_task(
                         # execute upgrade
                         new_unique_identifier = manifest.latest_package_identifier
 
-                        marketplace.record_install_plugin_event(new_unique_identifier)
+                        record_install_plugin_event(new_unique_identifier)
                         click.echo(
                             click.style(
                                 f"Upgrade plugin: {original_unique_identifier} -> {new_unique_identifier}",

api/tasks/remove_app_and_related_data_task.py

@@ -48,6 +48,7 @@ from models.workflow import (
     WorkflowArchiveLog,
 )
 from repositories.factory import DifyAPIRepositoryFactory
+from services.api_token_service import ApiTokenCache
 
 logger = logging.getLogger(__name__)
 
@@ -134,6 +135,12 @@ def _delete_app_mcp_servers(tenant_id: str, app_id: str):
 
 def _delete_app_api_tokens(tenant_id: str, app_id: str):
     def del_api_token(session, api_token_id: str):
+        # Fetch token details for cache invalidation
+        token_obj = session.query(ApiToken).where(ApiToken.id == api_token_id).first()
+        if token_obj:
+            # Invalidate cache before deletion
+            ApiTokenCache.delete(token_obj.token, token_obj.type)
+
         session.query(ApiToken).where(ApiToken.id == api_token_id).delete(synchronize_session=False)
 
     _delete_records(
@@ -259,8 +266,8 @@ def _delete_app_workflow_app_logs(tenant_id: str, app_id: str):
 
 
 def _delete_app_workflow_archive_logs(tenant_id: str, app_id: str):
-    def del_workflow_archive_log(workflow_archive_log_id: str):
-        db.session.query(WorkflowArchiveLog).where(WorkflowArchiveLog.id == workflow_archive_log_id).delete(
+    def del_workflow_archive_log(session, workflow_archive_log_id: str):
+        session.query(WorkflowArchiveLog).where(WorkflowArchiveLog.id == workflow_archive_log_id).delete(
             synchronize_session=False
         )
 
@@ -420,7 +427,7 @@ def delete_draft_variables_batch(app_id: str, batch_size: int = 1000) -> int:
     total_files_deleted = 0
 
     while True:
-        with session_factory.create_session() as session:
+        with session_factory.create_session() as session, session.begin():
             # Get a batch of draft variable IDs along with their file_ids
             query_sql = """
                 SELECT id, file_id FROM workflow_draft_variables

api/tests/integration_tests/libs/test_api_token_cache_integration.py

@@ -0,0 +1,375 @@
+"""
+Integration tests for API Token Cache with Redis.
+
+These tests require:
+- Redis server running
+- Test database configured
+"""
+
+import time
+from datetime import datetime, timedelta
+from unittest.mock import patch
+
+import pytest
+
+from extensions.ext_redis import redis_client
+from models.model import ApiToken
+from services.api_token_service import ApiTokenCache, CachedApiToken
+
+
+class TestApiTokenCacheRedisIntegration:
+    """Integration tests with real Redis."""
+
+    def setup_method(self):
+        """Setup test fixtures and clean Redis."""
+        self.test_token = "test-integration-token-123"
+        self.test_scope = "app"
+        self.cache_key = f"api_token:{self.test_scope}:{self.test_token}"
+
+        # Clean up any existing test data
+        self._cleanup()
+
+    def teardown_method(self):
+        """Cleanup test data from Redis."""
+        self._cleanup()
+
+    def _cleanup(self):
+        """Remove test data from Redis."""
+        try:
+            redis_client.delete(self.cache_key)
+            redis_client.delete(ApiTokenCache._make_tenant_index_key("test-tenant-id"))
+            redis_client.delete(ApiTokenCache.make_active_key(self.test_token, self.test_scope))
+        except Exception:
+            pass  # Ignore cleanup errors
+
+    def test_cache_set_and_get_with_real_redis(self):
+        """Test cache set and get operations with real Redis."""
+        from unittest.mock import MagicMock
+
+        mock_token = MagicMock()
+        mock_token.id = "test-id-123"
+        mock_token.app_id = "test-app-456"
+        mock_token.tenant_id = "test-tenant-789"
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = datetime.now()
+        mock_token.created_at = datetime.now() - timedelta(days=30)
+
+        # Set in cache
+        result = ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+        assert result is True
+
+        # Verify in Redis
+        cached_data = redis_client.get(self.cache_key)
+        assert cached_data is not None
+
+        # Get from cache
+        cached_token = ApiTokenCache.get(self.test_token, self.test_scope)
+        assert cached_token is not None
+        assert isinstance(cached_token, CachedApiToken)
+        assert cached_token.id == "test-id-123"
+        assert cached_token.app_id == "test-app-456"
+        assert cached_token.tenant_id == "test-tenant-789"
+        assert cached_token.type == "app"
+        assert cached_token.token == self.test_token
+
+    def test_cache_ttl_with_real_redis(self):
+        """Test cache TTL is set correctly."""
+        from unittest.mock import MagicMock
+
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = "test-tenant"
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+
+        ttl = redis_client.ttl(self.cache_key)
+        assert 595 <= ttl <= 600  # Should be around 600 seconds (10 minutes)
+
+    def test_cache_null_value_for_invalid_token(self):
+        """Test caching null value for invalid tokens."""
+        result = ApiTokenCache.set(self.test_token, self.test_scope, None)
+        assert result is True
+
+        cached_data = redis_client.get(self.cache_key)
+        assert cached_data == b"null"
+
+        cached_token = ApiTokenCache.get(self.test_token, self.test_scope)
+        assert cached_token is None
+
+        ttl = redis_client.ttl(self.cache_key)
+        assert 55 <= ttl <= 60
+
+    def test_cache_delete_with_real_redis(self):
+        """Test cache deletion with real Redis."""
+        from unittest.mock import MagicMock
+
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = "test-tenant"
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+        assert redis_client.exists(self.cache_key) == 1
+
+        result = ApiTokenCache.delete(self.test_token, self.test_scope)
+        assert result is True
+        assert redis_client.exists(self.cache_key) == 0
+
+    def test_tenant_index_creation(self):
+        """Test tenant index is created when caching token."""
+        from unittest.mock import MagicMock
+
+        tenant_id = "test-tenant-id"
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = tenant_id
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+
+        index_key = ApiTokenCache._make_tenant_index_key(tenant_id)
+        assert redis_client.exists(index_key) == 1
+
+        members = redis_client.smembers(index_key)
+        cache_keys = [m.decode("utf-8") if isinstance(m, bytes) else m for m in members]
+        assert self.cache_key in cache_keys
+
+    def test_invalidate_by_tenant_via_index(self):
+        """Test tenant-wide cache invalidation using index (fast path)."""
+        from unittest.mock import MagicMock
+
+        tenant_id = "test-tenant-id"
+
+        for i in range(3):
+            token_value = f"test-token-{i}"
+            mock_token = MagicMock()
+            mock_token.id = f"test-id-{i}"
+            mock_token.app_id = "test-app"
+            mock_token.tenant_id = tenant_id
+            mock_token.type = "app"
+            mock_token.token = token_value
+            mock_token.last_used_at = None
+            mock_token.created_at = datetime.now()
+
+            ApiTokenCache.set(token_value, "app", mock_token)
+
+        for i in range(3):
+            key = f"api_token:app:test-token-{i}"
+            assert redis_client.exists(key) == 1
+
+        result = ApiTokenCache.invalidate_by_tenant(tenant_id)
+        assert result is True
+
+        for i in range(3):
+            key = f"api_token:app:test-token-{i}"
+            assert redis_client.exists(key) == 0
+
+        assert redis_client.exists(ApiTokenCache._make_tenant_index_key(tenant_id)) == 0
+
+    def test_concurrent_cache_access(self):
+        """Test concurrent cache access doesn't cause issues."""
+        import concurrent.futures
+        from unittest.mock import MagicMock
+
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = "test-tenant"
+        mock_token.type = "app"
+        mock_token.token = self.test_token
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(self.test_token, self.test_scope, mock_token)
+
+        def get_from_cache():
+            return ApiTokenCache.get(self.test_token, self.test_scope)
+
+        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+            futures = [executor.submit(get_from_cache) for _ in range(50)]
+            results = [f.result() for f in concurrent.futures.as_completed(futures)]
+
+        assert len(results) == 50
+        assert all(r is not None for r in results)
+        assert all(isinstance(r, CachedApiToken) for r in results)
+
+
+class TestTokenUsageRecording:
+    """Tests for recording token usage in Redis (batch update approach)."""
+
+    def setup_method(self):
+        self.test_token = "test-usage-token"
+        self.test_scope = "app"
+        self.active_key = ApiTokenCache.make_active_key(self.test_token, self.test_scope)
+
+    def teardown_method(self):
+        try:
+            redis_client.delete(self.active_key)
+        except Exception:
+            pass
+
+    def test_record_token_usage_sets_redis_key(self):
+        """Test that record_token_usage writes an active key to Redis."""
+        from services.api_token_service import record_token_usage
+
+        record_token_usage(self.test_token, self.test_scope)
+
+        # Key should exist
+        assert redis_client.exists(self.active_key) == 1
+
+        # Value should be an ISO timestamp
+        value = redis_client.get(self.active_key)
+        if isinstance(value, bytes):
+            value = value.decode("utf-8")
+        datetime.fromisoformat(value)  # Should not raise
+
+    def test_record_token_usage_has_ttl(self):
+        """Test that active keys have a TTL as safety net."""
+        from services.api_token_service import record_token_usage
+
+        record_token_usage(self.test_token, self.test_scope)
+
+        ttl = redis_client.ttl(self.active_key)
+        assert 3595 <= ttl <= 3600  # ~1 hour
+
+    def test_record_token_usage_overwrites(self):
+        """Test that repeated calls overwrite the same key (no accumulation)."""
+        from services.api_token_service import record_token_usage
+
+        record_token_usage(self.test_token, self.test_scope)
+        first_value = redis_client.get(self.active_key)
+
+        time.sleep(0.01)  # Tiny delay so timestamp differs
+
+        record_token_usage(self.test_token, self.test_scope)
+        second_value = redis_client.get(self.active_key)
+
+        # Key count should still be 1 (overwritten, not accumulated)
+        assert redis_client.exists(self.active_key) == 1
+
+
+class TestEndToEndCacheFlow:
+    """End-to-end integration test for complete cache flow."""
+
+    @pytest.mark.usefixtures("db_session")
+    def test_complete_flow_cache_miss_then_hit(self, db_session):
+        """
+        Test complete flow:
+        1. First request (cache miss) -> query DB -> cache result
+        2. Second request (cache hit) -> return from cache
+        3. Verify Redis state
+        """
+        test_token_value = "test-e2e-token"
+        test_scope = "app"
+
+        test_token = ApiToken()
+        test_token.id = "test-e2e-id"
+        test_token.token = test_token_value
+        test_token.type = test_scope
+        test_token.app_id = "test-app"
+        test_token.tenant_id = "test-tenant"
+        test_token.last_used_at = None
+        test_token.created_at = datetime.now()
+
+        db_session.add(test_token)
+        db_session.commit()
+
+        try:
+            # Step 1: Cache miss - set token in cache
+            ApiTokenCache.set(test_token_value, test_scope, test_token)
+
+            cache_key = f"api_token:{test_scope}:{test_token_value}"
+            assert redis_client.exists(cache_key) == 1
+
+            # Step 2: Cache hit - get from cache
+            cached_token = ApiTokenCache.get(test_token_value, test_scope)
+            assert cached_token is not None
+            assert cached_token.id == test_token.id
+            assert cached_token.token == test_token_value
+
+            # Step 3: Verify tenant index
+            index_key = ApiTokenCache._make_tenant_index_key(test_token.tenant_id)
+            assert redis_client.exists(index_key) == 1
+            assert cache_key.encode() in redis_client.smembers(index_key)
+
+            # Step 4: Delete and verify cleanup
+            ApiTokenCache.delete(test_token_value, test_scope)
+            assert redis_client.exists(cache_key) == 0
+            assert cache_key.encode() not in redis_client.smembers(index_key)
+
+        finally:
+            db_session.delete(test_token)
+            db_session.commit()
+            redis_client.delete(f"api_token:{test_scope}:{test_token_value}")
+            redis_client.delete(ApiTokenCache._make_tenant_index_key(test_token.tenant_id))
+
+    def test_high_concurrency_simulation(self):
+        """Simulate high concurrency access to cache."""
+        import concurrent.futures
+        from unittest.mock import MagicMock
+
+        test_token_value = "test-concurrent-token"
+        test_scope = "app"
+
+        mock_token = MagicMock()
+        mock_token.id = "concurrent-id"
+        mock_token.app_id = "test-app"
+        mock_token.tenant_id = "test-tenant"
+        mock_token.type = test_scope
+        mock_token.token = test_token_value
+        mock_token.last_used_at = datetime.now()
+        mock_token.created_at = datetime.now()
+
+        ApiTokenCache.set(test_token_value, test_scope, mock_token)
+
+        try:
+
+            def read_cache():
+                return ApiTokenCache.get(test_token_value, test_scope)
+
+            start_time = time.time()
+            with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor:
+                futures = [executor.submit(read_cache) for _ in range(100)]
+                results = [f.result() for f in concurrent.futures.as_completed(futures)]
+            elapsed = time.time() - start_time
+
+            assert len(results) == 100
+            assert all(r is not None for r in results)
+
+            assert elapsed < 1.0, f"Too slow: {elapsed}s for 100 cache reads"
+
+        finally:
+            ApiTokenCache.delete(test_token_value, test_scope)
+            redis_client.delete(ApiTokenCache._make_tenant_index_key(mock_token.tenant_id))
+
+
+class TestRedisFailover:
+    """Test behavior when Redis is unavailable."""
+
+    @patch("services.api_token_service.redis_client")
+    def test_graceful_degradation_when_redis_fails(self, mock_redis):
+        """Test system degrades gracefully when Redis is unavailable."""
+        from redis import RedisError
+
+        mock_redis.get.side_effect = RedisError("Connection failed")
+        mock_redis.setex.side_effect = RedisError("Connection failed")
+
+        result_get = ApiTokenCache.get("test-token", "app")
+        assert result_get is None
+
+        result_set = ApiTokenCache.set("test-token", "app", None)
+        assert result_set is False

api/tests/integration_tests/tasks/test_remove_app_and_related_data_task.py

@@ -10,7 +10,10 @@ from models import Tenant
 from models.enums import CreatorUserRole
 from models.model import App, UploadFile
 from models.workflow import WorkflowDraftVariable, WorkflowDraftVariableFile
-from tasks.remove_app_and_related_data_task import _delete_draft_variables, delete_draft_variables_batch
+from tasks.remove_app_and_related_data_task import (
+    _delete_draft_variables,
+    delete_draft_variables_batch,
+)
 
 
 @pytest.fixture
@@ -297,12 +300,18 @@ class TestDeleteDraftVariablesWithOffloadIntegration:
     def test_delete_draft_variables_with_offload_data(self, mock_storage, setup_offload_test_data):
         data = setup_offload_test_data
         app_id = data["app"].id
+        upload_file_ids = [uf.id for uf in data["upload_files"]]
+        variable_file_ids = [vf.id for vf in data["variable_files"]]
         mock_storage.delete.return_value = None
 
         with session_factory.create_session() as session:
             draft_vars_before = session.query(WorkflowDraftVariable).filter_by(app_id=app_id).count()
-            var_files_before = session.query(WorkflowDraftVariableFile).count()
-            upload_files_before = session.query(UploadFile).count()
+            var_files_before = (
+                session.query(WorkflowDraftVariableFile)
+                .where(WorkflowDraftVariableFile.id.in_(variable_file_ids))
+                .count()
+            )
+            upload_files_before = session.query(UploadFile).where(UploadFile.id.in_(upload_file_ids)).count()
         assert draft_vars_before == 3
         assert var_files_before == 2
         assert upload_files_before == 2
@@ -315,8 +324,12 @@ class TestDeleteDraftVariablesWithOffloadIntegration:
         assert draft_vars_after == 0
 
         with session_factory.create_session() as session:
-            var_files_after = session.query(WorkflowDraftVariableFile).count()
-            upload_files_after = session.query(UploadFile).count()
+            var_files_after = (
+                session.query(WorkflowDraftVariableFile)
+                .where(WorkflowDraftVariableFile.id.in_(variable_file_ids))
+                .count()
+            )
+            upload_files_after = session.query(UploadFile).where(UploadFile.id.in_(upload_file_ids)).count()
         assert var_files_after == 0
         assert upload_files_after == 0
 
@@ -329,6 +342,8 @@ class TestDeleteDraftVariablesWithOffloadIntegration:
     def test_delete_draft_variables_storage_failure_continues_cleanup(self, mock_storage, setup_offload_test_data):
         data = setup_offload_test_data
         app_id = data["app"].id
+        upload_file_ids = [uf.id for uf in data["upload_files"]]
+        variable_file_ids = [vf.id for vf in data["variable_files"]]
         mock_storage.delete.side_effect = [Exception("Storage error"), None]
 
         deleted_count = delete_draft_variables_batch(app_id, batch_size=10)
@@ -339,8 +354,12 @@ class TestDeleteDraftVariablesWithOffloadIntegration:
         assert draft_vars_after == 0
 
         with session_factory.create_session() as session:
-            var_files_after = session.query(WorkflowDraftVariableFile).count()
-            upload_files_after = session.query(UploadFile).count()
+            var_files_after = (
+                session.query(WorkflowDraftVariableFile)
+                .where(WorkflowDraftVariableFile.id.in_(variable_file_ids))
+                .count()
+            )
+            upload_files_after = session.query(UploadFile).where(UploadFile.id.in_(upload_file_ids)).count()
         assert var_files_after == 0
         assert upload_files_after == 0
 
@@ -395,3 +414,275 @@ class TestDeleteDraftVariablesWithOffloadIntegration:
                 if app2_obj:
                     session.delete(app2_obj)
                 session.commit()
+
+
+class TestDeleteDraftVariablesSessionCommit:
+    """Test suite to verify session commit behavior in delete_draft_variables_batch."""
+
+    @pytest.fixture
+    def setup_offload_test_data(self, app_and_tenant):
+        """Create test data with offload files for session commit tests."""
+        from core.variables.types import SegmentType
+        from libs.datetime_utils import naive_utc_now
+
+        tenant, app = app_and_tenant
+
+        with session_factory.create_session() as session:
+            upload_file1 = UploadFile(
+                tenant_id=tenant.id,
+                storage_type="local",
+                key="test/file1.json",
+                name="file1.json",
+                size=1024,
+                extension="json",
+                mime_type="application/json",
+                created_by_role=CreatorUserRole.ACCOUNT,
+                created_by=str(uuid.uuid4()),
+                created_at=naive_utc_now(),
+                used=False,
+            )
+            upload_file2 = UploadFile(
+                tenant_id=tenant.id,
+                storage_type="local",
+                key="test/file2.json",
+                name="file2.json",
+                size=2048,
+                extension="json",
+                mime_type="application/json",
+                created_by_role=CreatorUserRole.ACCOUNT,
+                created_by=str(uuid.uuid4()),
+                created_at=naive_utc_now(),
+                used=False,
+            )
+            session.add(upload_file1)
+            session.add(upload_file2)
+            session.flush()
+
+            var_file1 = WorkflowDraftVariableFile(
+                tenant_id=tenant.id,
+                app_id=app.id,
+                user_id=str(uuid.uuid4()),
+                upload_file_id=upload_file1.id,
+                size=1024,
+                length=10,
+                value_type=SegmentType.STRING,
+            )
+            var_file2 = WorkflowDraftVariableFile(
+                tenant_id=tenant.id,
+                app_id=app.id,
+                user_id=str(uuid.uuid4()),
+                upload_file_id=upload_file2.id,
+                size=2048,
+                length=20,
+                value_type=SegmentType.OBJECT,
+            )
+            session.add(var_file1)
+            session.add(var_file2)
+            session.flush()
+
+            draft_var1 = WorkflowDraftVariable.new_node_variable(
+                app_id=app.id,
+                node_id="node_1",
+                name="large_var_1",
+                value=StringSegment(value="truncated..."),
+                node_execution_id=str(uuid.uuid4()),
+                file_id=var_file1.id,
+            )
+            draft_var2 = WorkflowDraftVariable.new_node_variable(
+                app_id=app.id,
+                node_id="node_2",
+                name="large_var_2",
+                value=StringSegment(value="truncated..."),
+                node_execution_id=str(uuid.uuid4()),
+                file_id=var_file2.id,
+            )
+            draft_var3 = WorkflowDraftVariable.new_node_variable(
+                app_id=app.id,
+                node_id="node_3",
+                name="regular_var",
+                value=StringSegment(value="regular_value"),
+                node_execution_id=str(uuid.uuid4()),
+            )
+            session.add(draft_var1)
+            session.add(draft_var2)
+            session.add(draft_var3)
+            session.commit()
+
+            data = {
+                "app": app,
+                "tenant": tenant,
+                "upload_files": [upload_file1, upload_file2],
+                "variable_files": [var_file1, var_file2],
+                "draft_variables": [draft_var1, draft_var2, draft_var3],
+            }
+
+        yield data
+
+        with session_factory.create_session() as session:
+            for table, ids in [
+                (WorkflowDraftVariable, [v.id for v in data["draft_variables"]]),
+                (WorkflowDraftVariableFile, [vf.id for vf in data["variable_files"]]),
+                (UploadFile, [uf.id for uf in data["upload_files"]]),
+            ]:
+                cleanup_query = delete(table).where(table.id.in_(ids)).execution_options(synchronize_session=False)
+                session.execute(cleanup_query)
+            session.commit()
+
+    @pytest.fixture
+    def setup_commit_test_data(self, app_and_tenant):
+        """Create test data for session commit tests."""
+        tenant, app = app_and_tenant
+        variable_ids: list[str] = []
+
+        with session_factory.create_session() as session:
+            variables = []
+            for i in range(10):
+                var = WorkflowDraftVariable.new_node_variable(
+                    app_id=app.id,
+                    node_id=f"node_{i}",
+                    name=f"var_{i}",
+                    value=StringSegment(value="test_value"),
+                    node_execution_id=str(uuid.uuid4()),
+                )
+                session.add(var)
+                variables.append(var)
+            session.commit()
+            variable_ids = [v.id for v in variables]
+
+        yield {
+            "app": app,
+            "tenant": tenant,
+            "variable_ids": variable_ids,
+        }
+
+        with session_factory.create_session() as session:
+            cleanup_query = (
+                delete(WorkflowDraftVariable)
+                .where(WorkflowDraftVariable.id.in_(variable_ids))
+                .execution_options(synchronize_session=False)
+            )
+            session.execute(cleanup_query)
+            session.commit()
+
+    def test_session_commit_is_called_after_each_batch(self, setup_commit_test_data):
+        """Test that session.begin() is used for automatic transaction management."""
+        data = setup_commit_test_data
+        app_id = data["app"].id
+
+        # Since session.begin() is used, the transaction is automatically committed
+        # when the with block exits successfully. We verify this by checking that
+        # data is actually persisted.
+        deleted_count = delete_draft_variables_batch(app_id, batch_size=3)
+
+        # Verify all data was deleted (proves transaction was committed)
+        with session_factory.create_session() as session:
+            remaining_count = session.query(WorkflowDraftVariable).filter_by(app_id=app_id).count()
+
+        assert deleted_count == 10
+        assert remaining_count == 0
+
+    def test_data_persisted_after_batch_deletion(self, setup_commit_test_data):
+        """Test that data is actually persisted to database after batch deletion with commits."""
+        data = setup_commit_test_data
+        app_id = data["app"].id
+        variable_ids = data["variable_ids"]
+
+        # Verify initial state
+        with session_factory.create_session() as session:
+            initial_count = session.query(WorkflowDraftVariable).filter_by(app_id=app_id).count()
+        assert initial_count == 10
+
+        # Perform deletion with small batch size to force multiple commits
+        deleted_count = delete_draft_variables_batch(app_id, batch_size=3)
+
+        assert deleted_count == 10
+
+        # Verify all data is deleted in a new session (proves commits worked)
+        with session_factory.create_session() as session:
+            final_count = session.query(WorkflowDraftVariable).filter_by(app_id=app_id).count()
+        assert final_count == 0
+
+        # Verify specific IDs are deleted
+        with session_factory.create_session() as session:
+            remaining_vars = (
+                session.query(WorkflowDraftVariable).where(WorkflowDraftVariable.id.in_(variable_ids)).count()
+            )
+        assert remaining_vars == 0
+
+    def test_session_commit_with_empty_dataset(self, setup_commit_test_data):
+        """Test session behavior when deleting from an empty dataset."""
+        nonexistent_app_id = str(uuid.uuid4())
+
+        # Should not raise any errors and should return 0
+        deleted_count = delete_draft_variables_batch(nonexistent_app_id, batch_size=10)
+        assert deleted_count == 0
+
+    def test_session_commit_with_single_batch(self, setup_commit_test_data):
+        """Test that commit happens correctly when all data fits in a single batch."""
+        data = setup_commit_test_data
+        app_id = data["app"].id
+
+        with session_factory.create_session() as session:
+            initial_count = session.query(WorkflowDraftVariable).filter_by(app_id=app_id).count()
+        assert initial_count == 10
+
+        # Delete all in a single batch
+        deleted_count = delete_draft_variables_batch(app_id, batch_size=100)
+        assert deleted_count == 10
+
+        # Verify data is persisted
+        with session_factory.create_session() as session:
+            final_count = session.query(WorkflowDraftVariable).filter_by(app_id=app_id).count()
+        assert final_count == 0
+
+    def test_invalid_batch_size_raises_error(self, setup_commit_test_data):
+        """Test that invalid batch size raises ValueError."""
+        data = setup_commit_test_data
+        app_id = data["app"].id
+
+        with pytest.raises(ValueError, match="batch_size must be positive"):
+            delete_draft_variables_batch(app_id, batch_size=0)
+
+        with pytest.raises(ValueError, match="batch_size must be positive"):
+            delete_draft_variables_batch(app_id, batch_size=-1)
+
+    @patch("extensions.ext_storage.storage")
+    def test_session_commit_with_offload_data_cleanup(self, mock_storage, setup_offload_test_data):
+        """Test that session commits correctly when cleaning up offload data."""
+        data = setup_offload_test_data
+        app_id = data["app"].id
+        upload_file_ids = [uf.id for uf in data["upload_files"]]
+        mock_storage.delete.return_value = None
+
+        # Verify initial state
+        with session_factory.create_session() as session:
+            draft_vars_before = session.query(WorkflowDraftVariable).filter_by(app_id=app_id).count()
+            var_files_before = (
+                session.query(WorkflowDraftVariableFile)
+                .where(WorkflowDraftVariableFile.id.in_([vf.id for vf in data["variable_files"]]))
+                .count()
+            )
+            upload_files_before = session.query(UploadFile).where(UploadFile.id.in_(upload_file_ids)).count()
+        assert draft_vars_before == 3
+        assert var_files_before == 2
+        assert upload_files_before == 2
+
+        # Delete variables with offload data
+        deleted_count = delete_draft_variables_batch(app_id, batch_size=10)
+        assert deleted_count == 3
+
+        # Verify all data is persisted (deleted) in new session
+        with session_factory.create_session() as session:
+            draft_vars_after = session.query(WorkflowDraftVariable).filter_by(app_id=app_id).count()
+            var_files_after = (
+                session.query(WorkflowDraftVariableFile)
+                .where(WorkflowDraftVariableFile.id.in_([vf.id for vf in data["variable_files"]]))
+                .count()
+            )
+            upload_files_after = session.query(UploadFile).where(UploadFile.id.in_(upload_file_ids)).count()
+        assert draft_vars_after == 0
+        assert var_files_after == 0
+        assert upload_files_after == 0
+
+        # Verify storage cleanup was called
+        assert mock_storage.delete.call_count == 2

api/tests/test_containers_integration_tests/services/test_account_service.py

@@ -1016,7 +1016,7 @@ class TestAccountService:
 
     def test_delete_account(self, db_session_with_containers, mock_external_service_dependencies):
         """
-        Test account deletion (should add task to queue).
+        Test account deletion (should add task to queue and sync to enterprise).
         """
         fake = Faker()
         email = fake.email()
@@ -1034,10 +1034,18 @@ class TestAccountService:
             password=password,
         )
 
-        with patch("services.account_service.delete_account_task") as mock_delete_task:
+        with (
+            patch("services.account_service.delete_account_task") as mock_delete_task,
+            patch("services.enterprise.account_deletion_sync.sync_account_deletion") as mock_sync,
+        ):
+            mock_sync.return_value = True
+
             # Delete account
             AccountService.delete_account(account)
 
+            # Verify sync was called
+            mock_sync.assert_called_once_with(account_id=account.id, source="account_deleted")
+
             # Verify task was added to queue
             mock_delete_task.delay.assert_called_once_with(account.id)
 
@@ -1716,7 +1724,7 @@ class TestTenantService:
 
     def test_remove_member_from_tenant_success(self, db_session_with_containers, mock_external_service_dependencies):
         """
-        Test successful member removal from tenant.
+        Test successful member removal from tenant (should sync to enterprise).
         """
         fake = Faker()
         tenant_name = fake.company()
@@ -1751,7 +1759,15 @@ class TestTenantService:
         TenantService.create_tenant_member(tenant, member_account, role="normal")
 
         # Remove member
-        TenantService.remove_member_from_tenant(tenant, member_account, owner_account)
+        with patch("services.enterprise.account_deletion_sync.sync_workspace_member_removal") as mock_sync:
+            mock_sync.return_value = True
+
+            TenantService.remove_member_from_tenant(tenant, member_account, owner_account)
+
+            # Verify sync was called
+            mock_sync.assert_called_once_with(
+                workspace_id=tenant.id, member_id=member_account.id, source="workspace_member_removed"
+            )
 
         # Verify member was removed
         from extensions.ext_database import db

api/tests/test_containers_integration_tests/tasks/test_document_indexing_update_task.py

@@ -0,0 +1,182 @@
+from unittest.mock import MagicMock, patch
+
+import pytest
+from faker import Faker
+
+from models import Account, Tenant, TenantAccountJoin, TenantAccountRole
+from models.dataset import Dataset, Document, DocumentSegment
+from tasks.document_indexing_update_task import document_indexing_update_task
+
+
+class TestDocumentIndexingUpdateTask:
+    @pytest.fixture
+    def mock_external_dependencies(self):
+        """Patch external collaborators used by the update task.
+        - IndexProcessorFactory.init_index_processor().clean(...)
+        - IndexingRunner.run([...])
+        """
+        with (
+            patch("tasks.document_indexing_update_task.IndexProcessorFactory") as mock_factory,
+            patch("tasks.document_indexing_update_task.IndexingRunner") as mock_runner,
+        ):
+            processor_instance = MagicMock()
+            mock_factory.return_value.init_index_processor.return_value = processor_instance
+
+            runner_instance = MagicMock()
+            mock_runner.return_value = runner_instance
+
+            yield {
+                "factory": mock_factory,
+                "processor": processor_instance,
+                "runner": mock_runner,
+                "runner_instance": runner_instance,
+            }
+
+    def _create_dataset_document_with_segments(self, db_session_with_containers, *, segment_count: int = 2):
+        fake = Faker()
+
+        # Account and tenant
+        account = Account(
+            email=fake.email(),
+            name=fake.name(),
+            interface_language="en-US",
+            status="active",
+        )
+        db_session_with_containers.add(account)
+        db_session_with_containers.commit()
+
+        tenant = Tenant(name=fake.company(), status="normal")
+        db_session_with_containers.add(tenant)
+        db_session_with_containers.commit()
+
+        join = TenantAccountJoin(
+            tenant_id=tenant.id,
+            account_id=account.id,
+            role=TenantAccountRole.OWNER,
+            current=True,
+        )
+        db_session_with_containers.add(join)
+        db_session_with_containers.commit()
+
+        # Dataset and document
+        dataset = Dataset(
+            tenant_id=tenant.id,
+            name=fake.company(),
+            description=fake.text(max_nb_chars=64),
+            data_source_type="upload_file",
+            indexing_technique="high_quality",
+            created_by=account.id,
+        )
+        db_session_with_containers.add(dataset)
+        db_session_with_containers.commit()
+
+        document = Document(
+            tenant_id=tenant.id,
+            dataset_id=dataset.id,
+            position=0,
+            data_source_type="upload_file",
+            batch="test_batch",
+            name=fake.file_name(),
+            created_from="upload_file",
+            created_by=account.id,
+            indexing_status="waiting",
+            enabled=True,
+            doc_form="text_model",
+        )
+        db_session_with_containers.add(document)
+        db_session_with_containers.commit()
+
+        # Segments
+        node_ids = []
+        for i in range(segment_count):
+            node_id = f"node-{i + 1}"
+            seg = DocumentSegment(
+                tenant_id=tenant.id,
+                dataset_id=dataset.id,
+                document_id=document.id,
+                position=i,
+                content=fake.text(max_nb_chars=32),
+                answer=None,
+                word_count=10,
+                tokens=5,
+                index_node_id=node_id,
+                status="completed",
+                created_by=account.id,
+            )
+            db_session_with_containers.add(seg)
+            node_ids.append(node_id)
+        db_session_with_containers.commit()
+
+        # Refresh to ensure ORM state
+        db_session_with_containers.refresh(dataset)
+        db_session_with_containers.refresh(document)
+
+        return dataset, document, node_ids
+
+    def test_cleans_segments_and_reindexes(self, db_session_with_containers, mock_external_dependencies):
+        dataset, document, node_ids = self._create_dataset_document_with_segments(db_session_with_containers)
+
+        # Act
+        document_indexing_update_task(dataset.id, document.id)
+
+        # Ensure we see committed changes from another session
+        db_session_with_containers.expire_all()
+
+        # Assert document status updated before reindex
+        updated = db_session_with_containers.query(Document).where(Document.id == document.id).first()
+        assert updated.indexing_status == "parsing"
+        assert updated.processing_started_at is not None
+
+        # Segments should be deleted
+        remaining = (
+            db_session_with_containers.query(DocumentSegment).where(DocumentSegment.document_id == document.id).count()
+        )
+        assert remaining == 0
+
+        # Assert index processor clean was called with expected args
+        clean_call = mock_external_dependencies["processor"].clean.call_args
+        assert clean_call is not None
+        args, kwargs = clean_call
+        # args[0] is a Dataset instance (from another session) — validate by id
+        assert getattr(args[0], "id", None) == dataset.id
+        # args[1] should contain our node_ids
+        assert set(args[1]) == set(node_ids)
+        assert kwargs.get("with_keywords") is True
+        assert kwargs.get("delete_child_chunks") is True
+
+        # Assert indexing runner invoked with the updated document
+        run_call = mock_external_dependencies["runner_instance"].run.call_args
+        assert run_call is not None
+        run_docs = run_call[0][0]
+        assert len(run_docs) == 1
+        first = run_docs[0]
+        assert getattr(first, "id", None) == document.id
+
+    def test_clean_error_is_logged_and_indexing_continues(self, db_session_with_containers, mock_external_dependencies):
+        dataset, document, node_ids = self._create_dataset_document_with_segments(db_session_with_containers)
+
+        # Force clean to raise; task should continue to indexing
+        mock_external_dependencies["processor"].clean.side_effect = Exception("boom")
+
+        document_indexing_update_task(dataset.id, document.id)
+
+        # Ensure we see committed changes from another session
+        db_session_with_containers.expire_all()
+
+        # Indexing should still be triggered
+        mock_external_dependencies["runner_instance"].run.assert_called_once()
+
+        # Segments should remain (since clean failed before DB delete)
+        remaining = (
+            db_session_with_containers.query(DocumentSegment).where(DocumentSegment.document_id == document.id).count()
+        )
+        assert remaining > 0
+
+    def test_document_not_found_noop(self, db_session_with_containers, mock_external_dependencies):
+        fake = Faker()
+        # Act with non-existent document id
+        document_indexing_update_task(dataset_id=fake.uuid4(), document_id=fake.uuid4())
+
+        # Neither processor nor runner should be called
+        mock_external_dependencies["processor"].clean.assert_not_called()
+        mock_external_dependencies["runner_instance"].run.assert_not_called()

api/tests/unit_tests/controllers/console/test_fastopenapi_feature.py

@@ -1,291 +0,0 @@
-import builtins
-import contextlib
-import importlib
-import sys
-from unittest.mock import MagicMock, PropertyMock, patch
-
-import pytest
-from flask import Flask
-from flask.views import MethodView
-from werkzeug.exceptions import Unauthorized
-
-from extensions import ext_fastopenapi
-from extensions.ext_database import db
-from services.feature_service import FeatureModel, SystemFeatureModel
-
-
-@pytest.fixture
-def app():
-    """
-    Creates a Flask application instance configured for testing.
-    """
-    app = Flask(__name__)
-    app.config["TESTING"] = True
-    app.config["SECRET_KEY"] = "test-secret"
-    app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
-
-    # Initialize the database with the app
-    db.init_app(app)
-
-    return app
-
-
-@pytest.fixture(autouse=True)
-def fix_method_view_issue(monkeypatch):
-    """
-    Automatic fixture to patch 'builtins.MethodView'.
-
-    Why this is needed:
-    The official legacy codebase contains a global patch in its initialization logic:
-        if not hasattr(builtins, "MethodView"):
-             builtins.MethodView = MethodView
-
-    Some dependencies (like ext_fastopenapi or older Flask extensions) might implicitly
-    rely on 'MethodView' being available in the global builtins namespace.
-
-    Refactoring Note:
-    While patching builtins is generally discouraged due to global side effects,
-    this fixture reproduces the production environment's state to ensure tests are realistic.
-    We use 'monkeypatch' to ensure that this change is undone after the test finishes,
-    keeping other tests isolated.
-    """
-    if not hasattr(builtins, "MethodView"):
-        # 'raising=False' allows us to set an attribute that doesn't exist yet
-        monkeypatch.setattr(builtins, "MethodView", MethodView, raising=False)
-
-
-# ------------------------------------------------------------------------------
-# Helper Functions for Fixture Complexity Reduction
-# ------------------------------------------------------------------------------
-
-
-def _create_isolated_router():
-    """
-    Creates a fresh, isolated router instance to prevent route pollution.
-    """
-    import controllers.fastopenapi
-
-    # Dynamically get the class type (e.g., FlaskRouter) to avoid hardcoding dependencies
-    RouterClass = type(controllers.fastopenapi.console_router)
-    return RouterClass()
-
-
-@contextlib.contextmanager
-def _patch_auth_and_router(temp_router):
-    """
-    Context manager that applies all necessary patches for:
-    1. The console_router (redirecting to our isolated temp_router)
-    2. Authentication decorators (disabling them with no-ops)
-    3. User/Account loaders (mocking authenticated state)
-    """
-
-    def noop(f):
-        return f
-
-    # We patch the SOURCE of the decorators/functions, not the destination module.
-    # This ensures that when 'controllers.console.feature' imports them, it gets the mocks.
-    with (
-        patch("controllers.fastopenapi.console_router", temp_router),
-        patch("extensions.ext_fastopenapi.console_router", temp_router),
-        patch("controllers.console.wraps.setup_required", side_effect=noop),
-        patch("libs.login.login_required", side_effect=noop),
-        patch("controllers.console.wraps.account_initialization_required", side_effect=noop),
-        patch("controllers.console.wraps.cloud_utm_record", side_effect=noop),
-        patch("libs.login.current_account_with_tenant", return_value=(MagicMock(), "tenant-id")),
-        patch("libs.login.current_user", MagicMock(is_authenticated=True)),
-    ):
-        # Explicitly reload ext_fastopenapi to ensure it uses the patched console_router
-        import extensions.ext_fastopenapi
-
-        importlib.reload(extensions.ext_fastopenapi)
-
-        yield
-
-
-def _force_reload_module(target_module: str, alias_module: str):
-    """
-    Forces a reload of the specified module and handles sys.modules aliasing.
-
-    Why reload?
-    Python decorators (like @route, @login_required) run at IMPORT time.
-    To apply our patches (mocks/no-ops) to these decorators, we must re-import
-    the module while the patches are active.
-
-    Why alias?
-    If 'ext_fastopenapi' imports the controller as 'api.controllers...', but we import
-    it as 'controllers...', Python treats them as two separate modules. This causes:
-    1. Double execution of decorators (registering routes twice -> AssertionError).
-    2. Type mismatch errors (Class A from module X is not Class A from module Y).
-
-    This function ensures both names point to the SAME loaded module instance.
-    """
-    # 1. Clean existing entries to force re-import
-    if target_module in sys.modules:
-        del sys.modules[target_module]
-    if alias_module in sys.modules:
-        del sys.modules[alias_module]
-
-    # 2. Import the module (triggering decorators with active patches)
-    module = importlib.import_module(target_module)
-
-    # 3. Alias the module in sys.modules to prevent double loading
-    sys.modules[alias_module] = sys.modules[target_module]
-
-    return module
-
-
-def _cleanup_modules(target_module: str, alias_module: str):
-    """
-    Removes the module and its alias from sys.modules to prevent side effects
-    on other tests.
-    """
-    if target_module in sys.modules:
-        del sys.modules[target_module]
-    if alias_module in sys.modules:
-        del sys.modules[alias_module]
-
-
-@pytest.fixture
-def mock_feature_module_env():
-    """
-    Sets up a mocked environment for the feature module.
-
-    This fixture orchestrates:
-    1. Creating an isolated router.
-    2. Patching authentication and global dependencies.
-    3. Reloading the controller module to apply patches to decorators.
-    4. cleaning up sys.modules afterwards.
-    """
-    target_module = "controllers.console.feature"
-    alias_module = "api.controllers.console.feature"
-
-    # 1. Prepare isolated router
-    temp_router = _create_isolated_router()
-
-    # 2. Apply patches
-    try:
-        with _patch_auth_and_router(temp_router):
-            # 3. Reload module to register routes on the temp_router
-            feature_module = _force_reload_module(target_module, alias_module)
-
-            yield feature_module
-
-    finally:
-        # 4. Teardown: Clean up sys.modules
-        _cleanup_modules(target_module, alias_module)
-
-
-# ------------------------------------------------------------------------------
-# Test Cases
-# ------------------------------------------------------------------------------
-
-
-@pytest.mark.parametrize(
-    ("url", "service_mock_path", "mock_model_instance", "json_key"),
-    [
-        (
-            "/console/api/features",
-            "controllers.console.feature.FeatureService.get_features",
-            FeatureModel(can_replace_logo=True),
-            "features",
-        ),
-        (
-            "/console/api/system-features",
-            "controllers.console.feature.FeatureService.get_system_features",
-            SystemFeatureModel(enable_marketplace=True),
-            "features",
-        ),
-    ],
-)
-def test_console_features_success(app, mock_feature_module_env, url, service_mock_path, mock_model_instance, json_key):
-    """
-    Tests that the feature APIs return a 200 OK status and correct JSON structure.
-    """
-    # Patch the service layer to return our mock model instance
-    with patch(service_mock_path, return_value=mock_model_instance):
-        # Initialize the API extension
-        ext_fastopenapi.init_app(app)
-
-        client = app.test_client()
-        response = client.get(url)
-
-    # Assertions
-    assert response.status_code == 200, f"Request failed with status {response.status_code}: {response.text}"
-
-    # Verify the JSON response matches the Pydantic model dump
-    expected_data = mock_model_instance.model_dump(mode="json")
-    assert response.get_json() == {json_key: expected_data}
-
-
-@pytest.mark.parametrize(
-    ("url", "service_mock_path"),
-    [
-        ("/console/api/features", "controllers.console.feature.FeatureService.get_features"),
-        ("/console/api/system-features", "controllers.console.feature.FeatureService.get_system_features"),
-    ],
-)
-def test_console_features_service_error(app, mock_feature_module_env, url, service_mock_path):
-    """
-    Tests how the application handles Service layer errors.
-
-    Note: When an exception occurs in the view, it is typically caught by the framework
-    (Flask or the OpenAPI wrapper) and converted to a 500 error response.
-    This test verifies that the application returns a 500 status code.
-    """
-    # Simulate a service failure
-    with patch(service_mock_path, side_effect=ValueError("Service Failure")):
-        ext_fastopenapi.init_app(app)
-        client = app.test_client()
-
-        # When an exception occurs in the view, it is typically caught by the framework
-        # (Flask or the OpenAPI wrapper) and converted to a 500 error response.
-        response = client.get(url)
-
-        assert response.status_code == 500
-        # Check if the error details are exposed in the response (depends on error handler config)
-        # We accept either generic 500 or the specific error message
-        assert "Service Failure" in response.text or "Internal Server Error" in response.text
-
-
-def test_system_features_unauthenticated(app, mock_feature_module_env):
-    """
-    Tests that /console/api/system-features endpoint works without authentication.
-
-    This test verifies the try-except block in get_system_features that handles
-    unauthenticated requests by passing is_authenticated=False to the service layer.
-    """
-    feature_module = mock_feature_module_env
-
-    # Override the behavior of the current_user mock
-    # The fixture patched 'libs.login.current_user', so 'controllers.console.feature.current_user'
-    # refers to that same Mock object.
-    mock_user = feature_module.current_user
-
-    # Simulate property access raising Unauthorized
-    # Note: We must reset side_effect if it was set, or set it here.
-    # The fixture initialized it as MagicMock(is_authenticated=True).
-    # We want type(mock_user).is_authenticated to raise Unauthorized.
-    type(mock_user).is_authenticated = PropertyMock(side_effect=Unauthorized)
-
-    # Patch the service layer for this specific test
-    with patch("controllers.console.feature.FeatureService.get_system_features") as mock_service:
-        # Setup mock service return value
-        mock_model = SystemFeatureModel(enable_marketplace=True)
-        mock_service.return_value = mock_model
-
-        # Initialize app
-        ext_fastopenapi.init_app(app)
-        client = app.test_client()
-
-        # Act
-        response = client.get("/console/api/system-features")
-
-        # Assert
-        assert response.status_code == 200, f"Request failed: {response.text}"
-
-        # Verify service was called with is_authenticated=False
-        mock_service.assert_called_once_with(is_authenticated=False)
-
-        # Verify response body
-        expected_data = mock_model.model_dump(mode="json")
-        assert response.get_json() == {"features": expected_data}

api/tests/unit_tests/controllers/console/test_fastopenapi_tags.py

@@ -1,222 +0,0 @@
-import builtins
-import contextlib
-import importlib
-import sys
-from types import SimpleNamespace
-from unittest.mock import MagicMock, patch
-
-import pytest
-from flask import Flask
-from flask.views import MethodView
-
-from extensions import ext_fastopenapi
-from extensions.ext_database import db
-
-
-@pytest.fixture
-def app():
-    app = Flask(__name__)
-    app.config["TESTING"] = True
-    app.config["SECRET_KEY"] = "test-secret"
-    app.config["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
-
-    db.init_app(app)
-
-    return app
-
-
-@pytest.fixture(autouse=True)
-def fix_method_view_issue(monkeypatch):
-    if not hasattr(builtins, "MethodView"):
-        monkeypatch.setattr(builtins, "MethodView", MethodView, raising=False)
-
-
-def _create_isolated_router():
-    import controllers.fastopenapi
-
-    router_class = type(controllers.fastopenapi.console_router)
-    return router_class()
-
-
-@contextlib.contextmanager
-def _patch_auth_and_router(temp_router):
-    def noop(func):
-        return func
-
-    default_user = MagicMock(has_edit_permission=True, is_dataset_editor=False)
-
-    with (
-        patch("controllers.fastopenapi.console_router", temp_router),
-        patch("extensions.ext_fastopenapi.console_router", temp_router),
-        patch("controllers.console.wraps.setup_required", side_effect=noop),
-        patch("libs.login.login_required", side_effect=noop),
-        patch("controllers.console.wraps.account_initialization_required", side_effect=noop),
-        patch("controllers.console.wraps.edit_permission_required", side_effect=noop),
-        patch("libs.login.current_account_with_tenant", return_value=(default_user, "tenant-id")),
-        patch("configs.dify_config.EDITION", "CLOUD"),
-    ):
-        import extensions.ext_fastopenapi
-
-        importlib.reload(extensions.ext_fastopenapi)
-
-        yield
-
-
-def _force_reload_module(target_module: str, alias_module: str):
-    if target_module in sys.modules:
-        del sys.modules[target_module]
-    if alias_module in sys.modules:
-        del sys.modules[alias_module]
-
-    module = importlib.import_module(target_module)
-    sys.modules[alias_module] = sys.modules[target_module]
-
-    return module
-
-
-def _dedupe_routes(router):
-    seen = set()
-    unique_routes = []
-    for path, method, endpoint in reversed(router.get_routes()):
-        key = (path, method, endpoint.__name__)
-        if key in seen:
-            continue
-        seen.add(key)
-        unique_routes.append((path, method, endpoint))
-    router._routes = list(reversed(unique_routes))
-
-
-def _cleanup_modules(target_module: str, alias_module: str):
-    if target_module in sys.modules:
-        del sys.modules[target_module]
-    if alias_module in sys.modules:
-        del sys.modules[alias_module]
-
-
-@pytest.fixture
-def mock_tags_module_env():
-    target_module = "controllers.console.tag.tags"
-    alias_module = "api.controllers.console.tag.tags"
-    temp_router = _create_isolated_router()
-
-    try:
-        with _patch_auth_and_router(temp_router):
-            tags_module = _force_reload_module(target_module, alias_module)
-            _dedupe_routes(temp_router)
-            yield tags_module
-    finally:
-        _cleanup_modules(target_module, alias_module)
-
-
-def test_list_tags_success(app: Flask, mock_tags_module_env):
-    # Arrange
-    tag = SimpleNamespace(id="tag-1", name="Alpha", type="app", binding_count=2)
-    with patch("controllers.console.tag.tags.TagService.get_tags", return_value=[tag]):
-        ext_fastopenapi.init_app(app)
-        client = app.test_client()
-
-        # Act
-        response = client.get("/console/api/tags?type=app&keyword=Alpha")
-
-    # Assert
-    assert response.status_code == 200
-    assert response.get_json() == [
-        {"id": "tag-1", "name": "Alpha", "type": "app", "binding_count": 2},
-    ]
-
-
-def test_create_tag_success(app: Flask, mock_tags_module_env):
-    # Arrange
-    tag = SimpleNamespace(id="tag-2", name="Beta", type="app")
-    with patch("controllers.console.tag.tags.TagService.save_tags", return_value=tag) as mock_save:
-        ext_fastopenapi.init_app(app)
-        client = app.test_client()
-
-        # Act
-        response = client.post("/console/api/tags", json={"name": "Beta", "type": "app"})
-
-    # Assert
-    assert response.status_code == 200
-    assert response.get_json() == {
-        "id": "tag-2",
-        "name": "Beta",
-        "type": "app",
-        "binding_count": 0,
-    }
-    mock_save.assert_called_once_with({"name": "Beta", "type": "app"})
-
-
-def test_update_tag_success(app: Flask, mock_tags_module_env):
-    # Arrange
-    tag = SimpleNamespace(id="tag-3", name="Gamma", type="app")
-    with (
-        patch("controllers.console.tag.tags.TagService.update_tags", return_value=tag) as mock_update,
-        patch("controllers.console.tag.tags.TagService.get_tag_binding_count", return_value=4),
-    ):
-        ext_fastopenapi.init_app(app)
-        client = app.test_client()
-
-        # Act
-        response = client.patch(
-            "/console/api/tags/11111111-1111-1111-1111-111111111111",
-            json={"name": "Gamma", "type": "app"},
-        )
-
-    # Assert
-    assert response.status_code == 200
-    assert response.get_json() == {
-        "id": "tag-3",
-        "name": "Gamma",
-        "type": "app",
-        "binding_count": 4,
-    }
-    mock_update.assert_called_once_with(
-        {"name": "Gamma", "type": "app"},
-        "11111111-1111-1111-1111-111111111111",
-    )
-
-
-def test_delete_tag_success(app: Flask, mock_tags_module_env):
-    # Arrange
-    with patch("controllers.console.tag.tags.TagService.delete_tag") as mock_delete:
-        ext_fastopenapi.init_app(app)
-        client = app.test_client()
-
-        # Act
-        response = client.delete("/console/api/tags/11111111-1111-1111-1111-111111111111")
-
-    # Assert
-    assert response.status_code == 204
-    mock_delete.assert_called_once_with("11111111-1111-1111-1111-111111111111")
-
-
-def test_create_tag_binding_success(app: Flask, mock_tags_module_env):
-    # Arrange
-    payload = {"tag_ids": ["tag-1", "tag-2"], "target_id": "target-1", "type": "app"}
-    with patch("controllers.console.tag.tags.TagService.save_tag_binding") as mock_bind:
-        ext_fastopenapi.init_app(app)
-        client = app.test_client()
-
-        # Act
-        response = client.post("/console/api/tag-bindings/create", json=payload)
-
-    # Assert
-    assert response.status_code == 200
-    assert response.get_json() == {"result": "success"}
-    mock_bind.assert_called_once_with(payload)
-
-
-def test_delete_tag_binding_success(app: Flask, mock_tags_module_env):
-    # Arrange
-    payload = {"tag_id": "tag-1", "target_id": "target-1", "type": "app"}
-    with patch("controllers.console.tag.tags.TagService.delete_tag_binding") as mock_unbind:
-        ext_fastopenapi.init_app(app)
-        client = app.test_client()
-
-        # Act
-        response = client.post("/console/api/tag-bindings/remove", json=payload)
-
-    # Assert
-    assert response.status_code == 200
-    assert response.get_json() == {"result": "success"}
-    mock_unbind.assert_called_once_with(payload)

api/tests/unit_tests/controllers/console/workspace/test_tool_providers.py

@@ -0,0 +1,364 @@
+"""Endpoint tests for controllers.console.workspace.tool_providers."""
+
+from __future__ import annotations
+
+import builtins
+import importlib
+from contextlib import contextmanager
+from types import ModuleType, SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+import pytest
+from flask import Flask
+from flask.views import MethodView
+
+if not hasattr(builtins, "MethodView"):
+    builtins.MethodView = MethodView  # type: ignore[attr-defined]
+
+
+_CONTROLLER_MODULE: ModuleType | None = None
+_WRAPS_MODULE: ModuleType | None = None
+_CONTROLLER_PATCHERS: list[patch] = []
+
+
+@contextmanager
+def _mock_db():
+    mock_session = SimpleNamespace(query=lambda *args, **kwargs: SimpleNamespace(first=lambda: True))
+    with patch("extensions.ext_database.db.session", mock_session):
+        yield
+
+
+@pytest.fixture
+def app() -> Flask:
+    flask_app = Flask(__name__)
+    flask_app.config["TESTING"] = True
+    return flask_app
+
+
+@pytest.fixture
+def controller_module(monkeypatch: pytest.MonkeyPatch):
+    module_name = "controllers.console.workspace.tool_providers"
+    global _CONTROLLER_MODULE
+    if _CONTROLLER_MODULE is None:
+
+        def _noop(func):
+            return func
+
+        patch_targets = [
+            ("libs.login.login_required", _noop),
+            ("controllers.console.wraps.setup_required", _noop),
+            ("controllers.console.wraps.account_initialization_required", _noop),
+            ("controllers.console.wraps.is_admin_or_owner_required", _noop),
+            ("controllers.console.wraps.enterprise_license_required", _noop),
+        ]
+        for target, value in patch_targets:
+            patcher = patch(target, value)
+            patcher.start()
+            _CONTROLLER_PATCHERS.append(patcher)
+        monkeypatch.setenv("DIFY_SETUP_READY", "true")
+        with _mock_db():
+            _CONTROLLER_MODULE = importlib.import_module(module_name)
+
+    module = _CONTROLLER_MODULE
+    monkeypatch.setattr(module, "jsonable_encoder", lambda payload: payload)
+
+    # Ensure decorators that consult deployment edition do not reach the database.
+    global _WRAPS_MODULE
+    wraps_module = importlib.import_module("controllers.console.wraps")
+    _WRAPS_MODULE = wraps_module
+    monkeypatch.setattr(module.dify_config, "EDITION", "CLOUD")
+    monkeypatch.setattr(wraps_module.dify_config, "EDITION", "CLOUD")
+
+    login_module = importlib.import_module("libs.login")
+    monkeypatch.setattr(login_module, "check_csrf_token", lambda *args, **kwargs: None)
+    return module
+
+
+def _mock_account(user_id: str = "user-123") -> SimpleNamespace:
+    return SimpleNamespace(id=user_id, status="active", is_authenticated=True, current_tenant_id=None)
+
+
+def _set_current_account(
+    monkeypatch: pytest.MonkeyPatch,
+    controller_module: ModuleType,
+    user: SimpleNamespace,
+    tenant_id: str,
+) -> None:
+    def _getter():
+        return user, tenant_id
+
+    user.current_tenant_id = tenant_id
+
+    monkeypatch.setattr(controller_module, "current_account_with_tenant", _getter)
+    if _WRAPS_MODULE is not None:
+        monkeypatch.setattr(_WRAPS_MODULE, "current_account_with_tenant", _getter)
+
+    login_module = importlib.import_module("libs.login")
+    monkeypatch.setattr(login_module, "_get_user", lambda: user)
+
+
+def test_tool_provider_list_calls_service_with_query(
+    app: Flask, controller_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-456")
+
+    service_mock = MagicMock(return_value=[{"provider": "builtin"}])
+    monkeypatch.setattr(controller_module.ToolCommonService, "list_tool_providers", service_mock)
+
+    with app.test_request_context("/workspaces/current/tool-providers?type=builtin"):
+        response = controller_module.ToolProviderListApi().get()
+
+    assert response == [{"provider": "builtin"}]
+    service_mock.assert_called_once_with(user.id, "tenant-456", "builtin")
+
+
+def test_builtin_provider_add_passes_payload(
+    app: Flask, controller_module: ModuleType, monkeypatch: pytest.MonkeyPatch
+):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-456")
+
+    service_mock = MagicMock(return_value={"status": "ok"})
+    monkeypatch.setattr(controller_module.BuiltinToolManageService, "add_builtin_tool_provider", service_mock)
+
+    payload = {
+        "credentials": {"api_key": "sk-test"},
+        "name": "MyTool",
+        "type": controller_module.CredentialType.API_KEY,
+    }
+
+    with app.test_request_context(
+        "/workspaces/current/tool-provider/builtin/openai/add",
+        method="POST",
+        json=payload,
+    ):
+        response = controller_module.ToolBuiltinProviderAddApi().post(provider="openai")
+
+    assert response == {"status": "ok"}
+    service_mock.assert_called_once_with(
+        user_id="user-123",
+        tenant_id="tenant-456",
+        provider="openai",
+        credentials={"api_key": "sk-test"},
+        name="MyTool",
+        api_type=controller_module.CredentialType.API_KEY,
+    )
+
+
+def test_builtin_provider_tools_get(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account("user-tenant-789")
+    _set_current_account(monkeypatch, controller_module, user, "tenant-789")
+
+    service_mock = MagicMock(return_value=[{"name": "tool-a"}])
+    monkeypatch.setattr(controller_module.BuiltinToolManageService, "list_builtin_tool_provider_tools", service_mock)
+    monkeypatch.setattr(controller_module, "jsonable_encoder", lambda payload: payload)
+
+    with app.test_request_context(
+        "/workspaces/current/tool-provider/builtin/my-provider/tools",
+        method="GET",
+    ):
+        response = controller_module.ToolBuiltinProviderListToolsApi().get(provider="my-provider")
+
+    assert response == [{"name": "tool-a"}]
+    service_mock.assert_called_once_with("tenant-789", "my-provider")
+
+
+def test_builtin_provider_info_get(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account("user-tenant-9")
+    _set_current_account(monkeypatch, controller_module, user, "tenant-9")
+    service_mock = MagicMock(return_value={"info": True})
+    monkeypatch.setattr(controller_module.BuiltinToolManageService, "get_builtin_tool_provider_info", service_mock)
+
+    with app.test_request_context("/info", method="GET"):
+        resp = controller_module.ToolBuiltinProviderInfoApi().get(provider="demo")
+
+    assert resp == {"info": True}
+    service_mock.assert_called_once_with("tenant-9", "demo")
+
+
+def test_builtin_provider_credentials_get(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account("user-tenant-cred")
+    _set_current_account(monkeypatch, controller_module, user, "tenant-cred")
+    service_mock = MagicMock(return_value=[{"cred": 1}])
+    monkeypatch.setattr(
+        controller_module.BuiltinToolManageService,
+        "get_builtin_tool_provider_credentials",
+        service_mock,
+    )
+
+    with app.test_request_context("/creds", method="GET"):
+        resp = controller_module.ToolBuiltinProviderGetCredentialsApi().get(provider="demo")
+
+    assert resp == [{"cred": 1}]
+    service_mock.assert_called_once_with(tenant_id="tenant-cred", provider_name="demo")
+
+
+def test_api_provider_remote_schema_get(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-10")
+    service_mock = MagicMock(return_value={"schema": "ok"})
+    monkeypatch.setattr(controller_module.ApiToolManageService, "get_api_tool_provider_remote_schema", service_mock)
+
+    with app.test_request_context("/remote?url=https://example.com/"):
+        resp = controller_module.ToolApiProviderGetRemoteSchemaApi().get()
+
+    assert resp == {"schema": "ok"}
+    service_mock.assert_called_once_with(user.id, "tenant-10", "https://example.com/")
+
+
+def test_api_provider_list_tools_get(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-11")
+    service_mock = MagicMock(return_value=[{"tool": "t"}])
+    monkeypatch.setattr(controller_module.ApiToolManageService, "list_api_tool_provider_tools", service_mock)
+
+    with app.test_request_context("/tools?provider=foo"):
+        resp = controller_module.ToolApiProviderListToolsApi().get()
+
+    assert resp == [{"tool": "t"}]
+    service_mock.assert_called_once_with(user.id, "tenant-11", "foo")
+
+
+def test_api_provider_get(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-12")
+    service_mock = MagicMock(return_value={"provider": "foo"})
+    monkeypatch.setattr(controller_module.ApiToolManageService, "get_api_tool_provider", service_mock)
+
+    with app.test_request_context("/get?provider=foo"):
+        resp = controller_module.ToolApiProviderGetApi().get()
+
+    assert resp == {"provider": "foo"}
+    service_mock.assert_called_once_with(user.id, "tenant-12", "foo")
+
+
+def test_builtin_provider_credentials_schema_get(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account("user-tenant-13")
+    _set_current_account(monkeypatch, controller_module, user, "tenant-13")
+    service_mock = MagicMock(return_value={"schema": True})
+    monkeypatch.setattr(
+        controller_module.BuiltinToolManageService,
+        "list_builtin_provider_credentials_schema",
+        service_mock,
+    )
+
+    with app.test_request_context("/schema", method="GET"):
+        resp = controller_module.ToolBuiltinProviderCredentialsSchemaApi().get(
+            provider="demo", credential_type="api-key"
+        )
+
+    assert resp == {"schema": True}
+    service_mock.assert_called_once()
+
+
+def test_workflow_provider_get_by_tool(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-wf")
+    tool_service = MagicMock(return_value={"wf": 1})
+    monkeypatch.setattr(
+        controller_module.WorkflowToolManageService,
+        "get_workflow_tool_by_tool_id",
+        tool_service,
+    )
+
+    tool_id = "00000000-0000-0000-0000-000000000001"
+    with app.test_request_context(f"/workflow?workflow_tool_id={tool_id}"):
+        resp = controller_module.ToolWorkflowProviderGetApi().get()
+
+    assert resp == {"wf": 1}
+    tool_service.assert_called_once_with(user.id, "tenant-wf", tool_id)
+
+
+def test_workflow_provider_get_by_app(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-wf2")
+    service_mock = MagicMock(return_value={"app": 1})
+    monkeypatch.setattr(
+        controller_module.WorkflowToolManageService,
+        "get_workflow_tool_by_app_id",
+        service_mock,
+    )
+
+    app_id = "00000000-0000-0000-0000-000000000002"
+    with app.test_request_context(f"/workflow?workflow_app_id={app_id}"):
+        resp = controller_module.ToolWorkflowProviderGetApi().get()
+
+    assert resp == {"app": 1}
+    service_mock.assert_called_once_with(user.id, "tenant-wf2", app_id)
+
+
+def test_workflow_provider_list_tools(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-wf3")
+    service_mock = MagicMock(return_value=[{"id": 1}])
+    monkeypatch.setattr(controller_module.WorkflowToolManageService, "list_single_workflow_tools", service_mock)
+
+    tool_id = "00000000-0000-0000-0000-000000000003"
+    with app.test_request_context(f"/workflow/tools?workflow_tool_id={tool_id}"):
+        resp = controller_module.ToolWorkflowProviderListToolApi().get()
+
+    assert resp == [{"id": 1}]
+    service_mock.assert_called_once_with(user.id, "tenant-wf3", tool_id)
+
+
+def test_builtin_tools_list(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-bt")
+
+    provider = SimpleNamespace(to_dict=lambda: {"name": "builtin"})
+    monkeypatch.setattr(
+        controller_module.BuiltinToolManageService,
+        "list_builtin_tools",
+        MagicMock(return_value=[provider]),
+    )
+
+    with app.test_request_context("/tools/builtin"):
+        resp = controller_module.ToolBuiltinListApi().get()
+
+    assert resp == [{"name": "builtin"}]
+
+
+def test_api_tools_list(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account("user-tenant-api")
+    _set_current_account(monkeypatch, controller_module, user, "tenant-api")
+
+    provider = SimpleNamespace(to_dict=lambda: {"name": "api"})
+    monkeypatch.setattr(
+        controller_module.ApiToolManageService,
+        "list_api_tools",
+        MagicMock(return_value=[provider]),
+    )
+
+    with app.test_request_context("/tools/api"):
+        resp = controller_module.ToolApiListApi().get()
+
+    assert resp == [{"name": "api"}]
+
+
+def test_workflow_tools_list(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account()
+    _set_current_account(monkeypatch, controller_module, user, "tenant-wf4")
+
+    provider = SimpleNamespace(to_dict=lambda: {"name": "wf"})
+    monkeypatch.setattr(
+        controller_module.WorkflowToolManageService,
+        "list_tenant_workflow_tools",
+        MagicMock(return_value=[provider]),
+    )
+
+    with app.test_request_context("/tools/workflow"):
+        resp = controller_module.ToolWorkflowListApi().get()
+
+    assert resp == [{"name": "wf"}]
+
+
+def test_tool_labels_list(app: Flask, controller_module, monkeypatch: pytest.MonkeyPatch):
+    user = _mock_account("user-label")
+    _set_current_account(monkeypatch, controller_module, user, "tenant-labels")
+    monkeypatch.setattr(controller_module.ToolLabelsService, "list_tool_labels", lambda: ["a", "b"])
+
+    with app.test_request_context("/tool-labels"):
+        resp = controller_module.ToolLabelsApi().get()
+
+    assert resp == ["a", "b"]

api/tests/unit_tests/core/workflow/nodes/template_transform/template_transform_node_spec.py

@@ -217,7 +217,6 @@ class TestTemplateTransformNode:
     @patch(
         "core.workflow.nodes.template_transform.template_transform_node.CodeExecutorJinja2TemplateRenderer.render_template"
     )
-    @patch("core.workflow.nodes.template_transform.template_transform_node.MAX_TEMPLATE_TRANSFORM_OUTPUT_LENGTH", 10)
     def test_run_output_length_exceeds_limit(
         self, mock_execute, basic_node_data, mock_graph, mock_graph_runtime_state, graph_init_params
     ):
@@ -231,6 +230,7 @@ class TestTemplateTransformNode:
             graph_init_params=graph_init_params,
             graph=mock_graph,
             graph_runtime_state=mock_graph_runtime_state,
+            max_output_length=10,
         )
 
         result = node._run()

api/tests/unit_tests/extensions/test_celery_ssl.py

@@ -132,6 +132,8 @@ class TestCelerySSLConfiguration:
         mock_config.WORKFLOW_SCHEDULE_MAX_DISPATCH_PER_TICK = 0
         mock_config.ENABLE_TRIGGER_PROVIDER_REFRESH_TASK = False
         mock_config.TRIGGER_PROVIDER_REFRESH_INTERVAL = 15
+        mock_config.ENABLE_API_TOKEN_LAST_USED_UPDATE_TASK = False
+        mock_config.API_TOKEN_LAST_USED_UPDATE_INTERVAL = 30
 
         with patch("extensions.ext_celery.dify_config", mock_config):
             from dify_app import DifyApp

api/tests/unit_tests/libs/test_api_token_cache.py

@@ -0,0 +1,250 @@
+"""
+Unit tests for API Token Cache module.
+"""
+
+import json
+from datetime import datetime
+from unittest.mock import MagicMock, patch
+
+from services.api_token_service import (
+    CACHE_KEY_PREFIX,
+    CACHE_NULL_TTL_SECONDS,
+    CACHE_TTL_SECONDS,
+    ApiTokenCache,
+    CachedApiToken,
+)
+
+
+class TestApiTokenCache:
+    """Test cases for ApiTokenCache class."""
+
+    def setup_method(self):
+        """Setup test fixtures."""
+        self.mock_token = MagicMock()
+        self.mock_token.id = "test-token-id-123"
+        self.mock_token.app_id = "test-app-id-456"
+        self.mock_token.tenant_id = "test-tenant-id-789"
+        self.mock_token.type = "app"
+        self.mock_token.token = "test-token-value-abc"
+        self.mock_token.last_used_at = datetime(2026, 2, 3, 10, 0, 0)
+        self.mock_token.created_at = datetime(2026, 1, 1, 0, 0, 0)
+
+    def test_make_cache_key(self):
+        """Test cache key generation."""
+        # Test with scope
+        key = ApiTokenCache._make_cache_key("my-token", "app")
+        assert key == f"{CACHE_KEY_PREFIX}:app:my-token"
+
+        # Test without scope
+        key = ApiTokenCache._make_cache_key("my-token", None)
+        assert key == f"{CACHE_KEY_PREFIX}:any:my-token"
+
+    def test_serialize_token(self):
+        """Test token serialization."""
+        serialized = ApiTokenCache._serialize_token(self.mock_token)
+        data = json.loads(serialized)
+
+        assert data["id"] == "test-token-id-123"
+        assert data["app_id"] == "test-app-id-456"
+        assert data["tenant_id"] == "test-tenant-id-789"
+        assert data["type"] == "app"
+        assert data["token"] == "test-token-value-abc"
+        assert data["last_used_at"] == "2026-02-03T10:00:00"
+        assert data["created_at"] == "2026-01-01T00:00:00"
+
+    def test_serialize_token_with_nulls(self):
+        """Test token serialization with None values."""
+        mock_token = MagicMock()
+        mock_token.id = "test-id"
+        mock_token.app_id = None
+        mock_token.tenant_id = None
+        mock_token.type = "dataset"
+        mock_token.token = "test-token"
+        mock_token.last_used_at = None
+        mock_token.created_at = datetime(2026, 1, 1, 0, 0, 0)
+
+        serialized = ApiTokenCache._serialize_token(mock_token)
+        data = json.loads(serialized)
+
+        assert data["app_id"] is None
+        assert data["tenant_id"] is None
+        assert data["last_used_at"] is None
+
+    def test_deserialize_token(self):
+        """Test token deserialization."""
+        cached_data = json.dumps(
+            {
+                "id": "test-id",
+                "app_id": "test-app",
+                "tenant_id": "test-tenant",
+                "type": "app",
+                "token": "test-token",
+                "last_used_at": "2026-02-03T10:00:00",
+                "created_at": "2026-01-01T00:00:00",
+            }
+        )
+
+        result = ApiTokenCache._deserialize_token(cached_data)
+
+        assert isinstance(result, CachedApiToken)
+        assert result.id == "test-id"
+        assert result.app_id == "test-app"
+        assert result.tenant_id == "test-tenant"
+        assert result.type == "app"
+        assert result.token == "test-token"
+        assert result.last_used_at == datetime(2026, 2, 3, 10, 0, 0)
+        assert result.created_at == datetime(2026, 1, 1, 0, 0, 0)
+
+    def test_deserialize_null_token(self):
+        """Test deserialization of null token (cached miss)."""
+        result = ApiTokenCache._deserialize_token("null")
+        assert result is None
+
+    def test_deserialize_invalid_json(self):
+        """Test deserialization with invalid JSON."""
+        result = ApiTokenCache._deserialize_token("invalid-json{")
+        assert result is None
+
+    @patch("services.api_token_service.redis_client")
+    def test_get_cache_hit(self, mock_redis):
+        """Test cache hit scenario."""
+        cached_data = json.dumps(
+            {
+                "id": "test-id",
+                "app_id": "test-app",
+                "tenant_id": "test-tenant",
+                "type": "app",
+                "token": "test-token",
+                "last_used_at": "2026-02-03T10:00:00",
+                "created_at": "2026-01-01T00:00:00",
+            }
+        ).encode("utf-8")
+        mock_redis.get.return_value = cached_data
+
+        result = ApiTokenCache.get("test-token", "app")
+
+        assert result is not None
+        assert isinstance(result, CachedApiToken)
+        assert result.app_id == "test-app"
+        mock_redis.get.assert_called_once_with(f"{CACHE_KEY_PREFIX}:app:test-token")
+
+    @patch("services.api_token_service.redis_client")
+    def test_get_cache_miss(self, mock_redis):
+        """Test cache miss scenario."""
+        mock_redis.get.return_value = None
+
+        result = ApiTokenCache.get("test-token", "app")
+
+        assert result is None
+        mock_redis.get.assert_called_once()
+
+    @patch("services.api_token_service.redis_client")
+    def test_set_valid_token(self, mock_redis):
+        """Test setting a valid token in cache."""
+        result = ApiTokenCache.set("test-token", "app", self.mock_token)
+
+        assert result is True
+        mock_redis.setex.assert_called_once()
+        args = mock_redis.setex.call_args[0]
+        assert args[0] == f"{CACHE_KEY_PREFIX}:app:test-token"
+        assert args[1] == CACHE_TTL_SECONDS
+
+    @patch("services.api_token_service.redis_client")
+    def test_set_null_token(self, mock_redis):
+        """Test setting a null token (cache penetration prevention)."""
+        result = ApiTokenCache.set("invalid-token", "app", None)
+
+        assert result is True
+        mock_redis.setex.assert_called_once()
+        args = mock_redis.setex.call_args[0]
+        assert args[0] == f"{CACHE_KEY_PREFIX}:app:invalid-token"
+        assert args[1] == CACHE_NULL_TTL_SECONDS
+        assert args[2] == b"null"
+
+    @patch("services.api_token_service.redis_client")
+    def test_delete_with_scope(self, mock_redis):
+        """Test deleting token cache with specific scope."""
+        result = ApiTokenCache.delete("test-token", "app")
+
+        assert result is True
+        mock_redis.delete.assert_called_once_with(f"{CACHE_KEY_PREFIX}:app:test-token")
+
+    @patch("services.api_token_service.redis_client")
+    def test_delete_without_scope(self, mock_redis):
+        """Test deleting token cache without scope (delete all)."""
+        # Mock scan_iter to return an iterator of keys
+        mock_redis.scan_iter.return_value = iter(
+            [
+                b"api_token:app:test-token",
+                b"api_token:dataset:test-token",
+            ]
+        )
+
+        result = ApiTokenCache.delete("test-token", None)
+
+        assert result is True
+        # Verify scan_iter was called with the correct pattern
+        mock_redis.scan_iter.assert_called_once()
+        call_args = mock_redis.scan_iter.call_args
+        assert call_args[1]["match"] == f"{CACHE_KEY_PREFIX}:*:test-token"
+
+        # Verify delete was called with all matched keys
+        mock_redis.delete.assert_called_once_with(
+            b"api_token:app:test-token",
+            b"api_token:dataset:test-token",
+        )
+
+    @patch("services.api_token_service.redis_client")
+    def test_redis_fallback_on_exception(self, mock_redis):
+        """Test Redis fallback when Redis is unavailable."""
+        from redis import RedisError
+
+        mock_redis.get.side_effect = RedisError("Connection failed")
+
+        result = ApiTokenCache.get("test-token", "app")
+
+        # Should return None (fallback) instead of raising exception
+        assert result is None
+
+
+class TestApiTokenCacheIntegration:
+    """Integration test scenarios."""
+
+    @patch("services.api_token_service.redis_client")
+    def test_full_cache_lifecycle(self, mock_redis):
+        """Test complete cache lifecycle: set -> get -> delete."""
+        # Setup mock token
+        mock_token = MagicMock()
+        mock_token.id = "id-123"
+        mock_token.app_id = "app-456"
+        mock_token.tenant_id = "tenant-789"
+        mock_token.type = "app"
+        mock_token.token = "token-abc"
+        mock_token.last_used_at = datetime(2026, 2, 3, 10, 0, 0)
+        mock_token.created_at = datetime(2026, 1, 1, 0, 0, 0)
+
+        # 1. Set token in cache
+        ApiTokenCache.set("token-abc", "app", mock_token)
+        assert mock_redis.setex.called
+
+        # 2. Simulate cache hit
+        cached_data = ApiTokenCache._serialize_token(mock_token)
+        mock_redis.get.return_value = cached_data  # bytes from model_dump_json().encode()
+
+        retrieved = ApiTokenCache.get("token-abc", "app")
+        assert retrieved is not None
+        assert isinstance(retrieved, CachedApiToken)
+
+        # 3. Delete from cache
+        ApiTokenCache.delete("token-abc", "app")
+        assert mock_redis.delete.called
+
+    @patch("services.api_token_service.redis_client")
+    def test_cache_penetration_prevention(self, mock_redis):
+        """Test that non-existent tokens are cached as null."""
+        # Set null token (cache miss)
+        ApiTokenCache.set("non-existent-token", "app", None)
+
+        args = mock_redis.setex.call_args[0]
+        assert args[2] == b"null"
+        assert args[1] == CACHE_NULL_TTL_SECONDS  # Shorter TTL for null values

api/tests/unit_tests/services/enterprise/test_account_deletion_sync.py

@@ -0,0 +1,276 @@
+"""Unit tests for account deletion synchronization.
+
+This test module verifies the enterprise account deletion sync functionality,
+including Redis queuing, error handling, and community vs enterprise behavior.
+"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+from redis import RedisError
+
+from services.enterprise.account_deletion_sync import (
+    _queue_task,
+    sync_account_deletion,
+    sync_workspace_member_removal,
+)
+
+
+class TestQueueTask:
+    """Unit tests for the _queue_task helper function."""
+
+    @pytest.fixture
+    def mock_redis_client(self):
+        """Mock redis_client for testing."""
+        with patch("services.enterprise.account_deletion_sync.redis_client") as mock_redis:
+            yield mock_redis
+
+    @pytest.fixture
+    def mock_uuid(self):
+        """Mock UUID generation for predictable task IDs."""
+        with patch("services.enterprise.account_deletion_sync.uuid.uuid4") as mock_uuid_gen:
+            mock_uuid_gen.return_value = MagicMock(hex="test-task-id-1234")
+            yield mock_uuid_gen
+
+    def test_queue_task_success(self, mock_redis_client, mock_uuid):
+        """Test successful task queueing to Redis."""
+        # Arrange
+        workspace_id = "ws-123"
+        member_id = "member-456"
+        source = "test_source"
+
+        # Act
+        result = _queue_task(workspace_id=workspace_id, member_id=member_id, source=source)
+
+        # Assert
+        assert result is True
+        mock_redis_client.lpush.assert_called_once()
+
+        # Verify the task payload structure
+        call_args = mock_redis_client.lpush.call_args[0]
+        assert call_args[0] == "enterprise:member:sync:queue"
+
+        import json
+
+        task_data = json.loads(call_args[1])
+        assert task_data["workspace_id"] == workspace_id
+        assert task_data["member_id"] == member_id
+        assert task_data["source"] == source
+        assert task_data["type"] == "sync_member_deletion_from_workspace"
+        assert task_data["retry_count"] == 0
+        assert "task_id" in task_data
+        assert "created_at" in task_data
+
+    def test_queue_task_redis_error(self, mock_redis_client, caplog):
+        """Test handling of Redis connection errors."""
+        # Arrange
+        mock_redis_client.lpush.side_effect = RedisError("Connection failed")
+
+        # Act
+        result = _queue_task(workspace_id="ws-123", member_id="member-456", source="test_source")
+
+        # Assert
+        assert result is False
+        assert "Failed to queue account deletion sync" in caplog.text
+
+    def test_queue_task_type_error(self, mock_redis_client, caplog):
+        """Test handling of JSON serialization errors."""
+        # Arrange
+        mock_redis_client.lpush.side_effect = TypeError("Cannot serialize")
+
+        # Act
+        result = _queue_task(workspace_id="ws-123", member_id="member-456", source="test_source")
+
+        # Assert
+        assert result is False
+        assert "Failed to queue account deletion sync" in caplog.text
+
+
+class TestSyncWorkspaceMemberRemoval:
+    """Unit tests for sync_workspace_member_removal function."""
+
+    @pytest.fixture
+    def mock_queue_task(self):
+        """Mock _queue_task for testing."""
+        with patch("services.enterprise.account_deletion_sync._queue_task") as mock_queue:
+            mock_queue.return_value = True
+            yield mock_queue
+
+    def test_sync_workspace_member_removal_enterprise_enabled(self, mock_queue_task):
+        """Test sync when ENTERPRISE_ENABLED is True."""
+        # Arrange
+        workspace_id = "ws-123"
+        member_id = "member-456"
+        source = "workspace_member_removed"
+
+        with patch("services.enterprise.account_deletion_sync.dify_config") as mock_config:
+            mock_config.ENTERPRISE_ENABLED = True
+
+            # Act
+            result = sync_workspace_member_removal(workspace_id=workspace_id, member_id=member_id, source=source)
+
+            # Assert
+            assert result is True
+            mock_queue_task.assert_called_once_with(workspace_id=workspace_id, member_id=member_id, source=source)
+
+    def test_sync_workspace_member_removal_enterprise_disabled(self, mock_queue_task):
+        """Test sync when ENTERPRISE_ENABLED is False (community edition)."""
+        # Arrange
+        with patch("services.enterprise.account_deletion_sync.dify_config") as mock_config:
+            mock_config.ENTERPRISE_ENABLED = False
+
+            # Act
+            result = sync_workspace_member_removal(workspace_id="ws-123", member_id="member-456", source="test_source")
+
+            # Assert
+            assert result is True
+            mock_queue_task.assert_not_called()
+
+    def test_sync_workspace_member_removal_queue_failure(self, mock_queue_task):
+        """Test handling of queue task failures."""
+        # Arrange
+        mock_queue_task.return_value = False
+
+        with patch("services.enterprise.account_deletion_sync.dify_config") as mock_config:
+            mock_config.ENTERPRISE_ENABLED = True
+
+            # Act
+            result = sync_workspace_member_removal(workspace_id="ws-123", member_id="member-456", source="test_source")
+
+            # Assert
+            assert result is False
+
+
+class TestSyncAccountDeletion:
+    """Unit tests for sync_account_deletion function."""
+
+    @pytest.fixture
+    def mock_db_session(self):
+        """Mock database session for testing."""
+        with patch("services.enterprise.account_deletion_sync.db.session") as mock_session:
+            yield mock_session
+
+    @pytest.fixture
+    def mock_queue_task(self):
+        """Mock _queue_task for testing."""
+        with patch("services.enterprise.account_deletion_sync._queue_task") as mock_queue:
+            mock_queue.return_value = True
+            yield mock_queue
+
+    def test_sync_account_deletion_enterprise_disabled(self, mock_db_session, mock_queue_task):
+        """Test sync when ENTERPRISE_ENABLED is False (community edition)."""
+        # Arrange
+        with patch("services.enterprise.account_deletion_sync.dify_config") as mock_config:
+            mock_config.ENTERPRISE_ENABLED = False
+
+            # Act
+            result = sync_account_deletion(account_id="acc-123", source="account_deleted")
+
+            # Assert
+            assert result is True
+            mock_db_session.query.assert_not_called()
+            mock_queue_task.assert_not_called()
+
+    def test_sync_account_deletion_multiple_workspaces(self, mock_db_session, mock_queue_task):
+        """Test sync for account with multiple workspace memberships."""
+        # Arrange
+        account_id = "acc-123"
+
+        # Mock workspace joins
+        mock_join1 = MagicMock()
+        mock_join1.tenant_id = "tenant-1"
+        mock_join2 = MagicMock()
+        mock_join2.tenant_id = "tenant-2"
+        mock_join3 = MagicMock()
+        mock_join3.tenant_id = "tenant-3"
+
+        mock_query = MagicMock()
+        mock_query.filter_by.return_value.all.return_value = [mock_join1, mock_join2, mock_join3]
+        mock_db_session.query.return_value = mock_query
+
+        with patch("services.enterprise.account_deletion_sync.dify_config") as mock_config:
+            mock_config.ENTERPRISE_ENABLED = True
+
+            # Act
+            result = sync_account_deletion(account_id=account_id, source="account_deleted")
+
+            # Assert
+            assert result is True
+            assert mock_queue_task.call_count == 3
+
+            # Verify each workspace was queued
+            mock_queue_task.assert_any_call(workspace_id="tenant-1", member_id=account_id, source="account_deleted")
+            mock_queue_task.assert_any_call(workspace_id="tenant-2", member_id=account_id, source="account_deleted")
+            mock_queue_task.assert_any_call(workspace_id="tenant-3", member_id=account_id, source="account_deleted")
+
+    def test_sync_account_deletion_no_workspaces(self, mock_db_session, mock_queue_task):
+        """Test sync for account with no workspace memberships."""
+        # Arrange
+        mock_query = MagicMock()
+        mock_query.filter_by.return_value.all.return_value = []
+        mock_db_session.query.return_value = mock_query
+
+        with patch("services.enterprise.account_deletion_sync.dify_config") as mock_config:
+            mock_config.ENTERPRISE_ENABLED = True
+
+            # Act
+            result = sync_account_deletion(account_id="acc-123", source="account_deleted")
+
+            # Assert
+            assert result is True
+            mock_queue_task.assert_not_called()
+
+    def test_sync_account_deletion_partial_failure(self, mock_db_session, mock_queue_task):
+        """Test sync when some tasks fail to queue."""
+        # Arrange
+        account_id = "acc-123"
+
+        # Mock workspace joins
+        mock_join1 = MagicMock()
+        mock_join1.tenant_id = "tenant-1"
+        mock_join2 = MagicMock()
+        mock_join2.tenant_id = "tenant-2"
+        mock_join3 = MagicMock()
+        mock_join3.tenant_id = "tenant-3"
+
+        mock_query = MagicMock()
+        mock_query.filter_by.return_value.all.return_value = [mock_join1, mock_join2, mock_join3]
+        mock_db_session.query.return_value = mock_query
+
+        # Mock queue_task to fail for second workspace
+        def queue_side_effect(workspace_id, member_id, source):
+            return workspace_id != "tenant-2"
+
+        mock_queue_task.side_effect = queue_side_effect
+
+        with patch("services.enterprise.account_deletion_sync.dify_config") as mock_config:
+            mock_config.ENTERPRISE_ENABLED = True
+
+            # Act
+            result = sync_account_deletion(account_id=account_id, source="account_deleted")
+
+            # Assert
+            assert result is False  # Should return False if any task fails
+            assert mock_queue_task.call_count == 3
+
+    def test_sync_account_deletion_all_failures(self, mock_db_session, mock_queue_task):
+        """Test sync when all tasks fail to queue."""
+        # Arrange
+        mock_join = MagicMock()
+        mock_join.tenant_id = "tenant-1"
+
+        mock_query = MagicMock()
+        mock_query.filter_by.return_value.all.return_value = [mock_join]
+        mock_db_session.query.return_value = mock_query
+
+        mock_queue_task.return_value = False
+
+        with patch("services.enterprise.account_deletion_sync.dify_config") as mock_config:
+            mock_config.ENTERPRISE_ENABLED = True
+
+            # Act
+            result = sync_account_deletion(account_id="acc-123", source="account_deleted")
+
+            # Assert
+            assert result is False
+            mock_queue_task.assert_called_once()

api/tests/unit_tests/tasks/test_document_indexing_sync_task.py

@@ -114,6 +114,21 @@ def mock_db_session():
         session = MagicMock()
         # Ensure tests can observe session.close() via context manager teardown
         session.close = MagicMock()
+        session.commit = MagicMock()
+
+        # Mock session.begin() context manager to auto-commit on exit
+        begin_cm = MagicMock()
+        begin_cm.__enter__.return_value = session
+
+        def _begin_exit_side_effect(*args, **kwargs):
+            # session.begin().__exit__() should commit if no exception
+            if args[0] is None:  # No exception
+                session.commit()
+
+        begin_cm.__exit__.side_effect = _begin_exit_side_effect
+        session.begin.return_value = begin_cm
+
+        # Mock create_session() context manager
         cm = MagicMock()
         cm.__enter__.return_value = session
 

api/tests/unit_tests/tasks/test_remove_app_and_related_data_task.py

@@ -350,7 +350,7 @@ class TestDeleteWorkflowArchiveLogs:
         mock_query.where.return_value = mock_delete_query
         mock_db.session.query.return_value = mock_query
 
-        delete_func("log-1")
+        delete_func(mock_db.session, "log-1")
 
         mock_db.session.query.assert_called_once_with(WorkflowArchiveLog)
         mock_query.where.assert_called_once()

api/uv.lock

@@ -1368,7 +1368,7 @@ wheels = [
 
 [[package]]
 name = "dify-api"
-version = "1.12.0"
+version = "1.12.1"
 source = { virtual = "." }
 dependencies = [
     { name = "aliyun-log-python-sdk" },
@@ -1707,7 +1707,7 @@ dev = [
     { name = "types-openpyxl", specifier = "~=3.1.5" },
     { name = "types-pexpect", specifier = "~=4.9.0" },
     { name = "types-protobuf", specifier = "~=5.29.1" },
-    { name = "types-psutil", specifier = "~=7.0.0" },
+    { name = "types-psutil", specifier = "~=7.2.2" },
     { name = "types-psycopg2", specifier = "~=2.9.21" },
     { name = "types-pygments", specifier = "~=2.19.0" },
     { name = "types-pymysql", specifier = "~=1.1.0" },
@@ -6508,11 +6508,11 @@ wheels = [
 
 [[package]]
 name = "types-psutil"
-version = "7.0.0.20251116"
+version = "7.2.2.20260130"
 source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/47/ec/c1e9308b91582cad1d7e7d3007fd003ef45a62c2500f8219313df5fc3bba/types_psutil-7.0.0.20251116.tar.gz", hash = "sha256:92b5c78962e55ce1ed7b0189901a4409ece36ab9fd50c3029cca7e681c606c8a", size = 22192, upload-time = "2025-11-16T03:10:32.859Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/69/14/fc5fb0a6ddfadf68c27e254a02ececd4d5c7fdb0efcb7e7e917a183497fb/types_psutil-7.2.2.20260130.tar.gz", hash = "sha256:15b0ab69c52841cf9ce3c383e8480c620a4d13d6a8e22b16978ebddac5590950", size = 26535, upload-time = "2026-01-30T03:58:14.116Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/c3/0e/11ba08a5375c21039ed5f8e6bba41e9452fb69f0e2f7ee05ed5cca2a2cdf/types_psutil-7.0.0.20251116-py3-none-any.whl", hash = "sha256:74c052de077c2024b85cd435e2cba971165fe92a5eace79cbeb821e776dbc047", size = 25376, upload-time = "2025-11-16T03:10:31.813Z" },
+    { url = "https://files.pythonhosted.org/packages/17/d7/60974b7e31545d3768d1770c5fe6e093182c3bfd819429b33133ba6b3e89/types_psutil-7.2.2.20260130-py3-none-any.whl", hash = "sha256:15523a3caa7b3ff03ac7f9b78a6470a59f88f48df1d74a39e70e06d2a99107da", size = 32876, upload-time = "2026-01-30T03:58:13.172Z" },
 ]
 
 [[package]]

dev/pytest/pytest_unit_tests.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -x
+set -euxo pipefail
 
 SCRIPT_DIR="$(dirname "$(realpath "$0")")"
 cd "$SCRIPT_DIR/../.."