fix(web): correct documentation links

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

.agents/skills/how-to-write-component/SKILL.md

@@ -25,6 +25,8 @@ Use this as the component decision guide for Dify web. Existing code is referenc
 - Search before adding UI, hooks, helpers, query utilities, or styling patterns. Reuse existing base components, feature components, hooks, utilities, and design styles when they fit.
 - Follow Dify's CSS-first Tailwind v4 contract from `packages/dify-ui/README.md` and `packages/dify-ui/AGENTS.md`. Prefer design-system tokens, utilities, and radius mappings over generic Tailwind choices.
 - Group feature code by workflow, route, or ownership area with route-aligned names: components, hooks, local types, query helpers, atoms, constants, tests, and small utilities should live near the code that changes with them.
+- For each feature module, keep a module-local `README.md` as a boundary note. Start with the module name, a brief one-sentence description, then split dependencies into `Internal Modules` and `External Modules` sections; keep both sections and write `None.` when one category is empty. `Internal Modules` lists modules inside the same overall feature using paths from that feature root, such as `shared/domain/runtime-status`; `External Modules` lists project modules outside the feature using paths from the web root without a `web/` prefix, such as `app/components/base/skeleton`. Omit npm packages, workspace package dependencies, and whitelisted plumbing modules. Do not copy caller-relative import paths into the README.
+- Module README whitelist: `@/service/client`, `@/next/*`.
 - Keep source/default selection, validation, dirty checks, and payload shaping close to the workflow that owns submit behavior. Do not hide flow-specific priority order, fallback behavior, or submit semantics in generic utilities.
 - Prefer direct conditionals for small branch-specific decisions, especially form source selection and request payload assembly.
 - Loading states for page sections, cards, lists, tables, forms, and drawers should be skeletons scoped to the content being loaded. Use spinners only for small inline busy indicators.

.github/CODEOWNERS

@@ -26,6 +26,9 @@
 /cli/ @GareArc
 /.github/workflows/cli-tests.yml @GareArc
 
+# E2E
+/e2e/ @lyzno1
+
 # Backend (default owner, more specific rules below will override)
 /api/ @QuantumGhost
 

.github/labeler.yml

@@ -7,3 +7,9 @@ web:
           - 'pnpm-lock.yaml'
           - 'pnpm-workspace.yaml'
           - '.nvmrc'
+
+e2e:
+  - changed-files:
+      - any-glob-to-any-file:
+          - 'e2e/**'
+          - '.github/workflows/web-e2e.yml'

api/clients/agent_backend/__init__.py

@@ -32,6 +32,8 @@ from clients.agent_backend.factory import create_agent_backend_run_client
 from clients.agent_backend.fake_client import FakeAgentBackendRunClient, FakeAgentBackendScenario
 from clients.agent_backend.request_builder import (
     AGENT_SOUL_PROMPT_LAYER_ID,
+    DIFY_CONFIG_LAYER_ID,
+    DIFY_CORE_TOOLS_LAYER_ID,
     DIFY_EXECUTION_CONTEXT_LAYER_ID,
     DIFY_KNOWLEDGE_BASE_LAYER_ID,
     DIFY_PLUGIN_TOOLS_LAYER_ID,
@@ -47,6 +49,8 @@ from clients.agent_backend.request_builder import (
 
 __all__ = [
     "AGENT_SOUL_PROMPT_LAYER_ID",
+    "DIFY_CONFIG_LAYER_ID",
+    "DIFY_CORE_TOOLS_LAYER_ID",
     "DIFY_EXECUTION_CONTEXT_LAYER_ID",
     "DIFY_KNOWLEDGE_BASE_LAYER_ID",
     "DIFY_PLUGIN_TOOLS_LAYER_ID",

api/clients/agent_backend/request_builder.py

@@ -20,6 +20,8 @@ from agenton.layers import ExitIntent
 from agenton_collections.layers.plain import PLAIN_PROMPT_LAYER_TYPE_ID, PromptLayerConfig
 from agenton_collections.layers.pydantic_ai import PYDANTIC_AI_HISTORY_LAYER_TYPE_ID
 from dify_agent.layers.ask_human import DIFY_ASK_HUMAN_LAYER_TYPE_ID, DifyAskHumanLayerConfig
+from dify_agent.layers.config import DIFY_CONFIG_LAYER_TYPE_ID, DifyConfigLayerConfig
+from dify_agent.layers.dify_core_tools import DIFY_CORE_TOOLS_LAYER_TYPE_ID, DifyCoreToolsLayerConfig
 from dify_agent.layers.dify_plugin import (
     DIFY_PLUGIN_LLM_LAYER_TYPE_ID,
     DIFY_PLUGIN_TOOLS_LAYER_TYPE_ID,
@@ -54,8 +56,10 @@ WORKFLOW_NODE_JOB_PROMPT_LAYER_ID = "workflow_node_job_prompt"
 WORKFLOW_USER_PROMPT_LAYER_ID = "workflow_user_prompt"
 AGENT_APP_USER_PROMPT_LAYER_ID = "agent_app_user_prompt"
 DIFY_EXECUTION_CONTEXT_LAYER_ID = "execution_context"
+DIFY_CONFIG_LAYER_ID = "config"
 DIFY_DRIVE_LAYER_ID = "drive"
 DIFY_PLUGIN_TOOLS_LAYER_ID = "tools"
+DIFY_CORE_TOOLS_LAYER_ID = "core_tools"
 DIFY_KNOWLEDGE_BASE_LAYER_ID = "knowledge"
 DIFY_ASK_HUMAN_LAYER_ID = "ask_human"
 DIFY_SHELL_LAYER_ID = "shell"
@@ -78,11 +82,26 @@ def _filter_snapshot_to_specs(
     return CompositorSessionSnapshot(schema_version=snapshot.schema_version, layers=filtered_layers)
 
 
-def _shell_layer_deps(*, include_drive: bool) -> dict[str, str]:
-    deps = {"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID}
-    if include_drive:
-        deps["drive"] = DIFY_DRIVE_LAYER_ID
-    return deps
+def _shell_layer_deps() -> dict[str, str]:
+    return {"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID}
+
+
+def _drive_layer_deps() -> dict[str, str]:
+    return {"shell": DIFY_SHELL_LAYER_ID}
+
+
+def _config_layer_deps() -> dict[str, str]:
+    return {"shell": DIFY_SHELL_LAYER_ID}
+
+
+def _shell_config_with_drive_ref(
+    shell_config: DifyShellLayerConfig | None,
+    drive_config: DifyDriveLayerConfig | None,
+) -> DifyShellLayerConfig:
+    config = shell_config or DifyShellLayerConfig()
+    if drive_config is None:
+        return config
+    return config.model_copy(update={"agent_stub_drive_ref": drive_config.drive_ref})
 
 
 class AgentBackendModelConfig(BaseModel):
@@ -148,7 +167,9 @@ class AgentBackendWorkflowNodeRunInput(BaseModel):
     idempotency_key: str | None = None
     output: AgentBackendOutputConfig | None = None
     tools: DifyPluginToolsLayerConfig | None = None
+    core_tools: DifyCoreToolsLayerConfig | None = None
     knowledge: DifyKnowledgeBaseLayerConfig | None = None
+    config_layer_config: DifyConfigLayerConfig | None = None
     # Drive Skills & Files declaration (dify.drive) — an index the agent pulls
     # through the back proxy, never inline content; see AGENT_DRIVE_MANIFEST_ENABLED.
     drive_config: DifyDriveLayerConfig | None = None
@@ -195,7 +216,9 @@ class AgentBackendAgentAppRunInput(BaseModel):
     idempotency_key: str | None = None
     output: AgentBackendOutputConfig | None = None
     tools: DifyPluginToolsLayerConfig | None = None
+    core_tools: DifyCoreToolsLayerConfig | None = None
     knowledge: DifyKnowledgeBaseLayerConfig | None = None
+    config_layer_config: DifyConfigLayerConfig | None = None
     # Drive Skills & Files declaration (dify.drive) — an index the agent pulls
     # through the back proxy, never inline content; see AGENT_DRIVE_MANIFEST_ENABLED.
     drive_config: DifyDriveLayerConfig | None = None
@@ -232,8 +255,9 @@ class AgentBackendRunRequestBuilder:
 
         Layer graph: optional Agent Soul system prompt → user prompt →
         execution context → optional history (multi-turn) → LLM → optional
-        plugin tools / knowledge search → optional structured output. Mirrors the workflow-node
-        layer ordering minus the workflow-job / previous-node prompt.
+        plugin-direct tools / core-routed tools / knowledge search →
+        optional structured output. Mirrors the workflow-node layer ordering
+        minus the workflow-job / previous-node prompt.
         """
         layers: list[RunLayerSpec] = []
         if run_input.agent_soul_prompt:
@@ -263,14 +287,42 @@ class AgentBackendRunRequestBuilder:
             ]
         )
 
+        include_shell = (
+            run_input.include_shell or run_input.config_layer_config is not None or run_input.drive_config is not None
+        )
+        if include_shell:
+            # Sandboxed bash workspace (dify.shell). It enters before config/drive
+            # so eager pulls materialize content in the same filesystem used by
+            # model commands.
+            layers.append(
+                RunLayerSpec(
+                    name=DIFY_SHELL_LAYER_ID,
+                    type=DIFY_SHELL_LAYER_TYPE_ID,
+                    deps=_shell_layer_deps(),
+                    metadata=run_input.metadata,
+                    config=_shell_config_with_drive_ref(run_input.shell_config, run_input.drive_config),
+                )
+            )
+
+        if run_input.config_layer_config is not None:
+            layers.append(
+                RunLayerSpec(
+                    name=DIFY_CONFIG_LAYER_ID,
+                    type=DIFY_CONFIG_LAYER_TYPE_ID,
+                    deps=_config_layer_deps(),
+                    metadata=run_input.metadata,
+                    config=run_input.config_layer_config,
+                )
+            )
+
         if run_input.drive_config is not None:
-            # Drive Skills & Files declaration (dify.drive): a config-only index;
-            # the agent pulls listed entries through the back proxy by drive_ref.
+            # Drive Skills & Files declaration (dify.drive): the catalog plus
+            # prompt-mentioned entries eagerly pulled through the shell layer.
             layers.append(
                 RunLayerSpec(
                     name=DIFY_DRIVE_LAYER_ID,
                     type=DIFY_DRIVE_LAYER_TYPE_ID,
-                    deps={"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID},
+                    deps=_drive_layer_deps(),
                     metadata=run_input.metadata,
                     config=run_input.drive_config,
                 )
@@ -312,7 +364,18 @@ class AgentBackendRunRequestBuilder:
                 )
             )
 
-        if run_input.knowledge is not None and run_input.knowledge.dataset_ids:
+        if run_input.core_tools is not None and run_input.core_tools.tools:
+            layers.append(
+                RunLayerSpec(
+                    name=DIFY_CORE_TOOLS_LAYER_ID,
+                    type=DIFY_CORE_TOOLS_LAYER_TYPE_ID,
+                    deps={"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID},
+                    metadata=run_input.metadata,
+                    config=run_input.core_tools,
+                )
+            )
+
+        if run_input.knowledge is not None and run_input.knowledge.sets:
             layers.append(
                 RunLayerSpec(
                     name=DIFY_KNOWLEDGE_BASE_LAYER_ID,
@@ -336,21 +399,6 @@ class AgentBackendRunRequestBuilder:
                 )
             )
 
-        if run_input.include_shell:
-            # Sandboxed bash workspace (dify.shell). Depends on execution_context
-            # so the agent server can mint per-command Agent Stub env, and on
-            # drive when present so that env points at /mnt/drive/<drive_ref>.
-            # shellctl connection itself is server-injected.
-            layers.append(
-                RunLayerSpec(
-                    name=DIFY_SHELL_LAYER_ID,
-                    type=DIFY_SHELL_LAYER_TYPE_ID,
-                    deps=_shell_layer_deps(include_drive=run_input.drive_config is not None),
-                    metadata=run_input.metadata,
-                    config=run_input.shell_config or DifyShellLayerConfig(),
-                )
-            )
-
         if run_input.output is not None:
             layers.append(
                 RunLayerSpec(
@@ -392,7 +440,8 @@ class AgentBackendRunRequestBuilder:
         non-plugin layer graph that produced the snapshot. Plugin layers
         (``dify.plugin.llm``, ``dify.plugin.tools``) are excluded from both the
         composition and the snapshot before submission because their configs
-        require credentials that are not persisted between runs.
+        may carry credentials or runtime-only declarations that are not
+        persisted between runs.
         """
         if not runtime_layer_specs:
             raise ValueError(
@@ -425,8 +474,9 @@ class AgentBackendRunRequestBuilder:
         """Build a workflow Agent Node run request without defining another wire schema.
 
         Layer graph mirrors the workflow surface: prompts → execution context →
-        optional drive/history → LLM → optional plugin tools / knowledge search
-        → optional auxiliary layers such as ask_human, shell, and structured output.
+        optional drive/history → LLM → optional plugin-direct tools /
+        core-routed tools / knowledge search → optional auxiliary layers such
+        as ask_human, shell, and structured output.
         """
         layers: list[RunLayerSpec] = []
         if run_input.agent_soul_prompt:
@@ -445,7 +495,7 @@ class AgentBackendRunRequestBuilder:
                     name=WORKFLOW_NODE_JOB_PROMPT_LAYER_ID,
                     type=PLAIN_PROMPT_LAYER_TYPE_ID,
                     metadata={**run_input.metadata, "origin": "workflow_node_job"},
-                    config=PromptLayerConfig(prefix=run_input.workflow_node_job_prompt),
+                    config=PromptLayerConfig(user=run_input.workflow_node_job_prompt),
                 ),
                 RunLayerSpec(
                     name=WORKFLOW_USER_PROMPT_LAYER_ID,
@@ -462,14 +512,42 @@ class AgentBackendRunRequestBuilder:
             ]
         )
 
+        include_shell = (
+            run_input.include_shell or run_input.config_layer_config is not None or run_input.drive_config is not None
+        )
+        if include_shell:
+            # Sandboxed bash workspace (dify.shell). It enters before drive so
+            # drive can materialize mentioned targets with `dify-agent drive pull`
+            # in the same shell-visible filesystem used by model commands.
+            layers.append(
+                RunLayerSpec(
+                    name=DIFY_SHELL_LAYER_ID,
+                    type=DIFY_SHELL_LAYER_TYPE_ID,
+                    deps=_shell_layer_deps(),
+                    metadata=run_input.metadata,
+                    config=_shell_config_with_drive_ref(run_input.shell_config, run_input.drive_config),
+                )
+            )
+
+        if run_input.config_layer_config is not None:
+            layers.append(
+                RunLayerSpec(
+                    name=DIFY_CONFIG_LAYER_ID,
+                    type=DIFY_CONFIG_LAYER_TYPE_ID,
+                    deps=_config_layer_deps(),
+                    metadata=run_input.metadata,
+                    config=run_input.config_layer_config,
+                )
+            )
+
         if run_input.drive_config is not None:
-            # Drive Skills & Files declaration (dify.drive): a config-only index;
-            # the agent pulls listed entries through the back proxy by drive_ref.
+            # Drive Skills & Files declaration (dify.drive): the catalog plus
+            # prompt-mentioned entries eagerly pulled through the shell layer.
             layers.append(
                 RunLayerSpec(
                     name=DIFY_DRIVE_LAYER_ID,
                     type=DIFY_DRIVE_LAYER_TYPE_ID,
-                    deps={"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID},
+                    deps=_drive_layer_deps(),
                     metadata=run_input.metadata,
                     config=run_input.drive_config,
                 )
@@ -513,7 +591,18 @@ class AgentBackendRunRequestBuilder:
                 )
             )
 
-        if run_input.knowledge is not None and run_input.knowledge.dataset_ids:
+        if run_input.core_tools is not None and run_input.core_tools.tools:
+            layers.append(
+                RunLayerSpec(
+                    name=DIFY_CORE_TOOLS_LAYER_ID,
+                    type=DIFY_CORE_TOOLS_LAYER_TYPE_ID,
+                    deps={"execution_context": DIFY_EXECUTION_CONTEXT_LAYER_ID},
+                    metadata=run_input.metadata,
+                    config=run_input.core_tools,
+                )
+            )
+
+        if run_input.knowledge is not None and run_input.knowledge.sets:
             layers.append(
                 RunLayerSpec(
                     name=DIFY_KNOWLEDGE_BASE_LAYER_ID,
@@ -537,21 +626,6 @@ class AgentBackendRunRequestBuilder:
                 )
             )
 
-        if run_input.include_shell:
-            # Sandboxed bash workspace (dify.shell). Depends on execution_context
-            # so the agent server can mint per-command Agent Stub env, and on
-            # drive when present so that env points at /mnt/drive/<drive_ref>.
-            # shellctl connection itself is server-injected.
-            layers.append(
-                RunLayerSpec(
-                    name=DIFY_SHELL_LAYER_ID,
-                    type=DIFY_SHELL_LAYER_TYPE_ID,
-                    deps=_shell_layer_deps(include_drive=run_input.drive_config is not None),
-                    metadata=run_input.metadata,
-                    config=run_input.shell_config or DifyShellLayerConfig(),
-                )
-            )
-
         if run_input.output is not None:
             layers.append(
                 RunLayerSpec(

api/commands/__init__.py

@@ -22,7 +22,7 @@ from .plugin import (
     setup_system_trigger_oauth_client,
     transform_datasource_credentials,
 )
-from .rbac import migrate_member_roles_to_rbac
+from .rbac import migrate_dataset_permissions_to_rbac, migrate_member_roles_to_rbac
 from .retention import (
     archive_workflow_runs,
     archive_workflow_runs_plan,
@@ -76,6 +76,7 @@ __all__ = [
     "legacy_model_types",
     "migrate_annotation_vector_database",
     "migrate_data_for_plugin",
+    "migrate_dataset_permissions_to_rbac",
     "migrate_knowledge_vector_database",
     "migrate_member_roles_to_rbac",
     "migrate_oss",

api/commands/data_migrate.py

@@ -7,6 +7,7 @@ from typing import cast
 
 import click
 
+from commands.rbac import migrate_dataset_permissions_to_rbac
 from extensions.ext_database import db
 from graphon.model_runtime.entities.model_entities import ModelType
 from services.legacy_model_type_migration import (
@@ -177,3 +178,4 @@ def legacy_model_types(
 
 
 data_migrate.add_command(legacy_model_types)
+data_migrate.add_command(migrate_dataset_permissions_to_rbac)

api/commands/rbac.py

@@ -1,11 +1,55 @@
 from __future__ import annotations
 
+import json
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor, as_completed
+
 import click
 from sqlalchemy import select
 
+from configs import dify_config
 from core.db.session_factory import session_factory
-from models import TenantAccountJoin, TenantAccountRole
-from services.enterprise.rbac_service import ListOption, RBACService
+from core.rbac import RBACResourceWhitelistScope
+from models import Dataset, DatasetPermission, DatasetPermissionEnum, TenantAccountJoin, TenantAccountRole
+from services.enterprise.rbac_service import ListOption, RBACService, ReplaceMemberBindings, ReplaceUserAccessPolicies
+
+_RBAC_DEFAULT_ACCESS_POLICY_ID = "default"
+
+_LEGACY_ROLE_TO_BUILTIN_TAG = {
+    TenantAccountRole.OWNER.value: "owner",
+    TenantAccountRole.ADMIN.value: "admin",
+    TenantAccountRole.EDITOR.value: "editor",
+    TenantAccountRole.NORMAL.value: "normal",
+    TenantAccountRole.DATASET_OPERATOR.value: "dataset_operator",
+}
+
+
+def _resolve_builtin_role_ids(tenant_id: str, operator_account_id: str) -> dict[str, str]:
+    """Resolve every legacy workspace role to the current tenant's builtin RBAC role id.
+
+    The migration replays the old `TenantAccountJoin.role` values onto the
+    RBAC member-role binding API. Builtin RBAC roles are tenant-scoped and
+    identified by runtime ids, so the command must look them up per tenant.
+    """
+    roles = RBACService.Roles.list(
+        tenant_id=tenant_id,
+        account_id=operator_account_id,
+        options=ListOption(page_number=1, results_per_page=100),
+    ).data
+    role_id_by_tag = {
+        role.role_tag: role.id
+        for role in roles
+        if role.is_builtin and role.category == "global_system_default" and role.role_tag
+    }
+    resolved: dict[str, str] = {}
+    for legacy_role, expected_builtin_tag in _LEGACY_ROLE_TO_BUILTIN_TAG.items():
+        role_id = role_id_by_tag.get(expected_builtin_tag)
+        if expected_builtin_tag == "dataset_operator" and not dify_config.DATASET_OPERATOR_ENABLED:
+            continue
+        if not role_id:
+            raise ValueError(f"Builtin RBAC role not found for tenant={tenant_id}, legacy_role={legacy_role}")
+        resolved[legacy_role] = role_id
+    return resolved
 
 
 def _resolve_builtin_role_id(tenant_id: str, operator_account_id: str, legacy_role: str) -> str:
@@ -15,26 +59,86 @@ def _resolve_builtin_role_id(tenant_id: str, operator_account_id: str, legacy_ro
     RBAC member-role binding API. Builtin RBAC roles are tenant-scoped and
     identified by runtime ids, so the command must look them up per tenant.
     """
-    expected_builtin_tag = {
-        TenantAccountRole.OWNER.value: "owner",
-        TenantAccountRole.ADMIN.value: "admin",
-        TenantAccountRole.EDITOR.value: "editor",
-        TenantAccountRole.NORMAL.value: "normal",
-        TenantAccountRole.DATASET_OPERATOR.value: "dataset_operator",
-    }.get(legacy_role)
-    if not expected_builtin_tag:
+    if legacy_role not in _LEGACY_ROLE_TO_BUILTIN_TAG:
         raise ValueError(f"Unsupported legacy workspace role: {legacy_role}")
 
-    roles = RBACService.Roles.list(
+    return _resolve_builtin_role_ids(tenant_id, operator_account_id)[legacy_role]
+
+
+def _iter_tenant_member_batches(
+    tenant_id: str | None,
+    *,
+    db_batch_size: int,
+    api_batch_size: int,
+) -> Iterator[tuple[str, str, list[tuple[str, str]]]]:
+    """Yield legacy member roles in tenant-scoped API-sized batches.
+
+    Rows are projected to primitive values and streamed from the database, so
+    the command never materializes every TenantAccountJoin ORM object. The
+    iterator only keeps one tenant's API-sized batches in memory while it
+    finds that tenant's owner account.
+    """
+    with session_factory.create_session() as session:
+        stmt = (
+            select(TenantAccountJoin.tenant_id, TenantAccountJoin.account_id, TenantAccountJoin.role)
+            .order_by(TenantAccountJoin.tenant_id.asc(), TenantAccountJoin.id.asc())
+            .execution_options(yield_per=db_batch_size)
+        )
+        if tenant_id:
+            stmt = stmt.where(TenantAccountJoin.tenant_id == tenant_id)
+
+        current_tenant_id: str | None = None
+        owner_account_id: str | None = None
+        batches: list[list[tuple[str, str]]] = []
+        batch: list[tuple[str, str]] = []
+
+        def flush_current_tenant() -> Iterator[tuple[str, str, list[tuple[str, str]]]]:
+            if current_tenant_id is None:
+                return
+            if batch:
+                batches.append(batch.copy())
+            if not owner_account_id:
+                raise ValueError(f"Workspace owner not found for tenant={current_tenant_id}")
+            for item in batches:
+                yield current_tenant_id, owner_account_id, item
+
+        for row in session.execute(stmt):
+            workspace_id = str(row.tenant_id)
+            if current_tenant_id is not None and workspace_id != current_tenant_id:
+                yield from flush_current_tenant()
+                owner_account_id = None
+                batches = []
+                batch = []
+            current_tenant_id = workspace_id
+            account_id = str(row.account_id)
+            role = str(row.role)
+            if role == TenantAccountRole.OWNER.value:
+                owner_account_id = account_id
+            batch.append((account_id, role))
+            if len(batch) >= api_batch_size:
+                batches.append(batch)
+                batch = []
+
+        yield from flush_current_tenant()
+
+
+def _member_already_has_role(current_roles_by_account_id: dict[str, set[str]], account_id: str, role_id: str) -> bool:
+    return current_roles_by_account_id.get(account_id) == {role_id}
+
+
+def _replace_member_role(
+    tenant_id: str,
+    operator_account_id: str,
+    member_account_id: str,
+    role_id: str,
+) -> str:
+    RBACService.MemberRoles.replace(
         tenant_id=tenant_id,
         account_id=operator_account_id,
-        options=ListOption(page_number=1, results_per_page=100),
-    ).data
-    for role in roles:
-        if role.is_builtin and role.category == "global_system_default" and role.role_tag == expected_builtin_tag:
-            return str(role.id)
-
-    raise ValueError(f"Builtin RBAC role not found for tenant={tenant_id}, legacy_role={legacy_role}")
+        member_account_id=member_account_id,
+        role_ids=[role_id],
+    )
+    return member_account_id
 
 
 @click.command(
@@ -42,7 +146,16 @@ def _resolve_builtin_role_id(tenant_id: str, operator_account_id: str, legacy_ro
 )
 @click.option("--tenant-id", help="Only migrate a single workspace.")
 @click.option("--dry-run", is_flag=True, default=False, help="Preview the migration without writing RBAC bindings.")
-def migrate_member_roles_to_rbac(tenant_id: str | None, dry_run: bool) -> None:
+@click.option("--db-batch-size", default=5000, show_default=True, help="Rows fetched per database batch.")
+@click.option("--api-batch-size", default=200, show_default=True, help="Members checked per RBAC batch_get call.")
+@click.option("--workers", default=1, show_default=True, help="Concurrent member role replace calls per tenant batch.")
+def migrate_member_roles_to_rbac(
+    tenant_id: str | None,
+    dry_run: bool,
+    db_batch_size: int,
+    api_batch_size: int,
+    workers: int,
+) -> None:
     """Backfill RBAC member-role bindings from legacy `TenantAccountJoin.role` data.
 
     This is an offline migration command for workspaces that already have
@@ -50,63 +163,322 @@ def migrate_member_roles_to_rbac(tenant_id: str | None, dry_run: bool) -> None:
     member-role binding store.
     """
     click.echo(click.style("Starting RBAC member-role migration.", fg="green"))
+    if workers < 1:
+        raise click.BadParameter("workers must be >= 1", param_hint="--workers")
 
-    with session_factory.create_session() as session:
-        stmt = select(TenantAccountJoin).order_by(TenantAccountJoin.tenant_id.asc(), TenantAccountJoin.id.asc())
-        if tenant_id:
-            stmt = stmt.where(TenantAccountJoin.tenant_id == tenant_id)
+    tenant_count = 0
+    scanned_count = 0
+    skipped_count = 0
+    migrated_count = 0
+    current_tenant_id: str | None = None
+    role_ids_by_legacy_role: dict[str, str] = {}
 
-        joins = list(session.scalars(stmt).all())
+    for workspace_id, owner_account_id, batch in _iter_tenant_member_batches(
+        tenant_id,
+        db_batch_size=db_batch_size,
+        api_batch_size=api_batch_size,
+    ):
+        scanned_count += len(batch)
+        if workspace_id != current_tenant_id:
+            tenant_count += 1
+            current_tenant_id = workspace_id
+            role_ids_by_legacy_role = _resolve_builtin_role_ids(workspace_id, owner_account_id)
+            click.echo(f"tenant={workspace_id}")
 
-    if not joins:
+        current_roles_by_account_id: dict[str, set[str]] = {}
+        if not dry_run:
+            current_roles = RBACService.MemberRoles.batch_get(
+                tenant_id=workspace_id,
+                account_id=owner_account_id,
+                member_account_ids=[account_id for account_id, _ in batch],
+            )
+            current_roles_by_account_id = {
+                item.account_id: {str(role.id) for role in item.roles} for item in current_roles
+            }
+
+        replace_jobs: list[tuple[str, str]] = []
+        for member_account_id, legacy_role in batch:
+            resolved_role_id = role_ids_by_legacy_role.get(legacy_role)
+            if not resolved_role_id:
+                raise ValueError(f"Unsupported legacy workspace role: {legacy_role}")
+
+            if dry_run:
+                click.echo(
+                    f"tenant={workspace_id} member={member_account_id} "
+                    f"legacy_role={legacy_role} -> rbac_role_id={resolved_role_id}"
+                )
+                continue
+
+            if _member_already_has_role(current_roles_by_account_id, member_account_id, resolved_role_id):
+                skipped_count += 1
+                continue
+
+            replace_jobs.append((member_account_id, resolved_role_id))
+
+        if replace_jobs:
+            if workers == 1:
+                for member_account_id, resolved_role_id in replace_jobs:
+                    _replace_member_role(workspace_id, owner_account_id, member_account_id, resolved_role_id)
+                    migrated_count += 1
+            else:
+                with ThreadPoolExecutor(max_workers=workers) as executor:
+                    futures = [
+                        executor.submit(
+                            _replace_member_role,
+                            workspace_id,
+                            owner_account_id,
+                            member_account_id,
+                            resolved_role_id,
+                        )
+                        for member_account_id, resolved_role_id in replace_jobs
+                    ]
+                    for future in as_completed(futures):
+                        future.result()
+                        migrated_count += 1
+
+        if scanned_count % 10000 == 0:
+            click.echo(
+                f"progress scanned={scanned_count} migrated={migrated_count} skipped={skipped_count}",
+                err=True,
+            )
+
+    if scanned_count == 0:
         click.echo(click.style("No workspace members found for migration.", fg="yellow"))
         return
 
-    owner_account_by_tenant: dict[str, str] = {}
-    resolved_role_ids: dict[tuple[str, str], str] = {}
-    migrated_count = 0
-
-    for join in joins:
-        workspace_id = str(join.tenant_id)
-        member_account_id = str(join.account_id)
-        legacy_role = str(join.role)
-
-        if workspace_id not in owner_account_by_tenant:
-            owner_join = next(
-                (
-                    item
-                    for item in joins
-                    if str(item.tenant_id) == workspace_id and str(item.role) == TenantAccountRole.OWNER.value
-                ),
-                None,
-            )
-            if not owner_join:
-                raise ValueError(f"Workspace owner not found for tenant={workspace_id}")
-            owner_account_by_tenant[workspace_id] = str(owner_join.account_id)
-
-        operator_account_id = owner_account_by_tenant[workspace_id]
-        cache_key = (workspace_id, legacy_role)
-        if cache_key not in resolved_role_ids:
-            resolved_role_ids[cache_key] = _resolve_builtin_role_id(workspace_id, operator_account_id, legacy_role)
-
-        resolved_role_id = resolved_role_ids[cache_key]
+    if dry_run:
         click.echo(
-            f"tenant={workspace_id} member={member_account_id} "
-            f"legacy_role={legacy_role} -> rbac_role_id={resolved_role_id}"
+            click.style(
+                f"Dry run completed. Scanned {scanned_count} members across {tenant_count} tenants. "
+                "No RBAC bindings were written.",
+                fg="yellow",
+            )
+        )
+    else:
+        click.echo(
+            click.style(
+                f"RBAC member-role migration completed. Scanned {scanned_count} members across {tenant_count} tenants, "
+                f"migrated {migrated_count}, skipped {skipped_count} already up-to-date.",
+                fg="green",
+            )
         )
 
-        if dry_run:
-            continue
 
-        RBACService.MemberRoles.replace(
-            tenant_id=workspace_id,
-            account_id=operator_account_id,
-            member_account_id=member_account_id,
-            role_ids=[resolved_role_id],
-        )
-        migrated_count += 1
+def _dataset_permission_enum(permission: DatasetPermissionEnum | str | None) -> DatasetPermissionEnum:
+    if permission is None:
+        return DatasetPermissionEnum.ONLY_ME
+    try:
+        return DatasetPermissionEnum(permission)
+    except ValueError as exc:
+        raise ValueError(f"Unsupported legacy dataset permission: {permission}") from exc
+
+
+def _rbac_dataset_scope_for_legacy_permission(permission: DatasetPermissionEnum) -> RBACResourceWhitelistScope:
+    if permission is DatasetPermissionEnum.ALL_TEAM:
+        return RBACResourceWhitelistScope.ALL
+    if permission in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.PARTIAL_TEAM}:
+        return RBACResourceWhitelistScope.SPECIFIC
+    raise ValueError(f"Unsupported legacy dataset permission: {permission}")
+
+
+def _emit_dataset_permission_migration_event(payload: dict[str, object]) -> None:
+    click.echo(json.dumps(payload, sort_keys=True))
+
+
+@click.command(
+    "rbac-migrate-dataset-permissions",
+    help=(
+        "Migrate legacy dataset permission scopes and partial members into RBAC dataset access bindings. "
+        "Side effect: replacing each dataset whitelist clears existing per-user policy bindings; "
+        "the command then recreates legacy partial-member default bindings."
+    ),
+)
+@click.option("--tenant-id", help="Only migrate datasets in a single workspace.")
+@click.option("--dataset-id", help="Only migrate a single dataset.")
+@click.option("--batch-size", default=500, show_default=True, type=click.IntRange(min=1))
+@click.option(
+    "--dry-run/--apply",
+    default=True,
+    show_default=True,
+    help="Preview the migration without writing RBAC bindings. Use --apply to write changes.",
+)
+def migrate_dataset_permissions_to_rbac(
+    tenant_id: str | None,
+    dataset_id: str | None,
+    batch_size: int,
+    dry_run: bool,
+) -> None:
+    """Backfill RBAC dataset access config from legacy `Dataset.permission`.
+
+    Legacy mapping:
+    - all_team_members -> RBAC dataset whitelist scope "all"
+    - partial_members  -> RBAC dataset whitelist scope "specific" plus each partial member gets the
+      virtual default policy
+    - only_me          -> RBAC dataset whitelist scope "specific" with no member policy bindings
+
+    The command replaces each dataset's RBAC whitelist scope first. RBAC clears
+    existing per-user policy bindings during that replace, then this command
+    recreates the legacy partial-member default bindings. Re-running it is
+    therefore idempotent for a dataset's current legacy configuration.
+    """
+    click.echo(click.style("Starting RBAC dataset permission migration.", fg="green"))
+
+    scanned_count = 0
+    scope_migrated_count = 0
+    user_policy_migrated_count = 0
+    partial_dataset_count = 0
+
+    last_dataset_id: str | None = None
+    while True:
+        with session_factory.create_session() as session:
+            stmt = (
+                select(Dataset.id, Dataset.tenant_id, Dataset.permission, Dataset.created_by)
+                .order_by(Dataset.id.asc())
+                .limit(batch_size)
+            )
+            if tenant_id:
+                stmt = stmt.where(Dataset.tenant_id == tenant_id)
+            if dataset_id:
+                stmt = stmt.where(Dataset.id == dataset_id)
+            if last_dataset_id:
+                stmt = stmt.where(Dataset.id > last_dataset_id)
+
+            dataset_rows = list(session.execute(stmt).all())
+            if not dataset_rows:
+                break
+
+            dataset_ids = [str(row.id) for row in dataset_rows]
+            partial_members_by_dataset_id: dict[str, list[str]] = {item: [] for item in dataset_ids}
+            permission_rows = session.execute(
+                select(DatasetPermission.dataset_id, DatasetPermission.account_id).where(
+                    DatasetPermission.dataset_id.in_(dataset_ids)
+                )
+            ).all()
+            for row in permission_rows:
+                partial_members_by_dataset_id[str(row.dataset_id)].append(str(row.account_id))
+
+        for dataset in dataset_rows:
+            workspace_id = str(dataset.tenant_id)
+            current_dataset_id = str(dataset.id)
+            operator_account_id = str(dataset.created_by)
+            permission_value = _dataset_permission_enum(dataset.permission)
+            scope = _rbac_dataset_scope_for_legacy_permission(permission_value)
+            partial_member_ids = sorted(set(partial_members_by_dataset_id[current_dataset_id]))
+            should_bind_partial_members = permission_value is DatasetPermissionEnum.PARTIAL_TEAM
+
+            click.echo(
+                f"tenant={workspace_id} dataset={current_dataset_id} "
+                f"operator={operator_account_id} "
+                f"legacy_permission={permission_value} -> rbac_scope={scope} "
+                f"partial_members={len(partial_member_ids) if should_bind_partial_members else 0}"
+            )
+
+            scanned_count += 1
+            replace_whitelist_payload = ReplaceMemberBindings(scope=scope)
+            if dry_run:
+                _emit_dataset_permission_migration_event(
+                    {
+                        "event": "dataset_permission_migration_proposed_change",
+                        "action": "replace_whitelist",
+                        "dry_run": True,
+                        "tenant_id": workspace_id,
+                        "dataset_id": current_dataset_id,
+                        "operator_account_id": operator_account_id,
+                        "before": {
+                            "legacy_dataset_permission": permission_value.value,
+                            "legacy_partial_member_ids": partial_member_ids if should_bind_partial_members else [],
+                        },
+                        "after": {
+                            "rbac_whitelist_scope": scope.value,
+                        },
+                        "call": {
+                            "method": "RBACService.DatasetAccess.replace_whitelist",
+                            "kwargs": {
+                                "tenant_id": workspace_id,
+                                "account_id": operator_account_id,
+                                "dataset_id": current_dataset_id,
+                                "payload": replace_whitelist_payload.model_dump(mode="json"),
+                            },
+                        },
+                    }
+                )
+            if not dry_run:
+                RBACService.DatasetAccess.replace_whitelist(
+                    tenant_id=workspace_id,
+                    account_id=operator_account_id,
+                    dataset_id=current_dataset_id,
+                    payload=replace_whitelist_payload,
+                )
+                scope_migrated_count += 1
+
+            if should_bind_partial_members:
+                partial_dataset_count += 1
+                for member_account_id in partial_member_ids:
+                    replace_user_access_policies_payload = ReplaceUserAccessPolicies(
+                        access_policy_ids=[_RBAC_DEFAULT_ACCESS_POLICY_ID],
+                    )
+                    if dry_run:
+                        _emit_dataset_permission_migration_event(
+                            {
+                                "event": "dataset_permission_migration_proposed_change",
+                                "action": "replace_user_access_policies",
+                                "dry_run": True,
+                                "tenant_id": workspace_id,
+                                "dataset_id": current_dataset_id,
+                                "operator_account_id": operator_account_id,
+                                "target_account_id": member_account_id,
+                                "before": {
+                                    "legacy_dataset_permission": permission_value.value,
+                                    "legacy_partial_member_id": member_account_id,
+                                },
+                                "after": {
+                                    "rbac_user_access_policy_ids": [_RBAC_DEFAULT_ACCESS_POLICY_ID],
+                                },
+                                "call": {
+                                    "method": "RBACService.DatasetAccess.replace_user_access_policies",
+                                    "kwargs": {
+                                        "tenant_id": workspace_id,
+                                        "account_id": operator_account_id,
+                                        "dataset_id": current_dataset_id,
+                                        "target_account_id": member_account_id,
+                                        "payload": replace_user_access_policies_payload.model_dump(mode="json"),
+                                    },
+                                },
+                            }
+                        )
+                        continue
+                    RBACService.DatasetAccess.replace_user_access_policies(
+                        tenant_id=workspace_id,
+                        account_id=operator_account_id,
+                        dataset_id=current_dataset_id,
+                        target_account_id=member_account_id,
+                        payload=replace_user_access_policies_payload,
+                    )
+                    user_policy_migrated_count += 1
+
+        last_dataset_id = dataset_ids[-1]
+
+        if dataset_id:
+            break
+
+    if scanned_count == 0:
+        click.echo(click.style("No datasets found for migration.", fg="yellow"))
+        return
 
     if dry_run:
-        click.echo(click.style("Dry run completed. No RBAC bindings were written.", fg="yellow"))
+        click.echo(
+            click.style(
+                f"Dry run completed. Scanned {scanned_count} datasets; "
+                f"{partial_dataset_count} partial-member datasets would be migrated.",
+                fg="yellow",
+            )
+        )
     else:
-        click.echo(click.style(f"RBAC member-role migration completed. Migrated {migrated_count} members.", fg="green"))
+        click.echo(
+            click.style(
+                "RBAC dataset permission migration completed. "
+                f"Scanned {scanned_count} datasets, migrated {scope_migrated_count} scopes, "
+                f"wrote {user_policy_migrated_count} user default-policy bindings.",
+                fg="green",
+            )
+        )

api/commands/retention.py

@@ -35,6 +35,12 @@ class WorkflowRunArchiveTenantPlan(TypedDict):
     unpaid_tenant_ids: list[str]
 
 
+def _normalize_utc_datetime(value: datetime.datetime) -> datetime.datetime:
+    if value.tzinfo is None:
+        return value.replace(tzinfo=datetime.UTC)
+    return value.astimezone(datetime.UTC)
+
+
 def _parse_tenant_prefixes(prefixes: str | None) -> list[str]:
     if not prefixes:
         return []
@@ -156,11 +162,16 @@ def _resolve_archive_time_range(
             raise click.UsageError("Choose either day offsets or explicit dates, not both.")
         if from_days_ago <= to_days_ago:
             raise click.UsageError("--from-days-ago must be greater than --to-days-ago.")
-        now = datetime.datetime.now()
+        now = datetime.datetime.now(datetime.UTC)
         start_from = now - datetime.timedelta(days=from_days_ago)
         end_before = now - datetime.timedelta(days=to_days_ago)
         before_days = 0
 
+    if start_from is not None:
+        start_from = _normalize_utc_datetime(start_from)
+    if end_before is not None:
+        end_before = _normalize_utc_datetime(end_before)
+
     if start_from and end_before and start_from >= end_before:
         raise click.UsageError("--start-from must be earlier than --end-before.")
 
@@ -402,6 +413,13 @@ def archive_workflow_runs_plan(
             fg="white",
         )
     )
+    click.echo(
+        click.style(
+            "fixed_archive_window="
+            f"{start_from.isoformat() if start_from else 'unbounded'},{plan_end_before.isoformat()}",
+            fg="white",
+        )
+    )
     click.echo("tenant_prefix,total_tenants,workflow_runs,workflow_node_executions,paid_tenants,unpaid_tenants")
     for row in rows:
         click.echo(
@@ -451,7 +469,7 @@ def archive_workflow_runs_plan(
     default=None,
     help="Archive runs created before this timestamp (UTC if no timezone).",
 )
-@click.option("--batch-size", default=100, show_default=True, help="Maximum workflow runs per archive bundle.")
+@click.option("--batch-size", default=10000, show_default=True, help="Maximum workflow runs per archive bundle.")
 @click.option(
     "--workers",
     default=1,
@@ -521,6 +539,7 @@ def archive_workflow_runs(
         )
     )
 
+    uses_relative_window = start_from is None and end_before is None
     try:
         before_days, start_from, end_before = _resolve_archive_time_range(
             before_days=before_days,
@@ -546,6 +565,14 @@ def archive_workflow_runs(
     if delete_after_archive:
         click.echo(click.style("delete-after-archive is not supported by bundle archive.", fg="red"))
         return
+    if uses_relative_window:
+        click.echo(
+            click.style(
+                "Relative archive windows are evaluated at command start. For multi-day prefix/shard rollout, "
+                "reuse absolute --start-from/--end-before values from archive-workflow-runs-plan.",
+                fg="yellow",
+            )
+        )
 
     try:
         tenant_plan = _resolve_archive_tenant_ids_from_plan(

api/configs/enterprise/__init__.py

@@ -16,7 +16,7 @@ class EnterpriseFeatureConfig(BaseSettings):
 
     CAN_REPLACE_LOGO: bool = Field(
         description="Allow customization of the enterprise logo.",
-        default=True,
+        default=False,
     )
 
     ENTERPRISE_REQUEST_TIMEOUT: int = Field(
@@ -34,6 +34,12 @@ class EnterpriseFeatureConfig(BaseSettings):
         default=False,
     )
 
+    ENTERPRISE_RBAC_REQUEST_TIMEOUT: int = Field(
+        ge=1,
+        description="Maximum timeout in seconds for inner RBAC requests.",
+        default=30,
+    )
+
 
 class EnterpriseTelemetryConfig(BaseSettings):
     """

api/configs/extra/agent_backend_config.py

@@ -36,8 +36,8 @@ class AgentBackendConfig(BaseSettings):
         description=(
             "Inject the dify.drive layer (Skills & Files drive manifest declaration) "
             "into Agent runs. The declaration is an index only — the agent backend "
-            "pulls the actual SKILL.md / files through the back proxy. Keep it off "
-            "until the agent backend registers the dify.drive layer type."
+            "pulls the actual SKILL.md / files through the back proxy. Set this to "
+            "false only when temporarily rolling back the drive integration."
         ),
-        default=False,
+        default=True,
     )

api/controllers/common/fields.py

@@ -183,6 +183,7 @@ class Site(BaseModel):
     description: str | None = None
     copyright: str | None = None
     privacy_policy: str | None = None
+    input_placeholder: str | None = None
     custom_disclaimer: str | None = None
     default_language: str
     show_workflow_steps: bool

api/controllers/console/__init__.py

@@ -54,6 +54,7 @@ from .app import (
     agent_app_access,
     agent_app_feature,
     agent_app_sandbox,
+    agent_config_inspector,
     agent_drive_inspector,
     annotation,
     app,
@@ -157,6 +158,7 @@ __all__ = [
     "agent_app_feature",
     "agent_app_sandbox",
     "agent_composer",
+    "agent_config_inspector",
     "agent_drive_inspector",
     "agent_providers",
     "agent_roster",

api/controllers/console/agent/app_helpers.py

@@ -6,5 +6,15 @@ from services.agent.roster_service import AgentRosterService
 
 
 def resolve_agent_app_model(*, tenant_id: str, agent_id: UUID) -> App:
-    """Resolve the hidden Agent App backing an Agent Console resource."""
+    """Resolve a roster Agent's public Agent App."""
     return AgentRosterService(db.session).get_agent_app_model(tenant_id=tenant_id, agent_id=str(agent_id))
+
+
+def resolve_agent_runtime_app_model(*, tenant_id: str, agent_id: UUID) -> App:
+    """Resolve the App that backs an Agent runtime surface.
+
+    This accepts both roster Agent Apps and workflow-only inline Agents with a
+    hidden backing App.
+    """
+
+    return AgentRosterService(db.session).get_agent_runtime_app_model(tenant_id=tenant_id, agent_id=str(agent_id))

api/controllers/console/agent/composer.py

@@ -1,10 +1,10 @@
 from uuid import UUID
 
+from flask import request
 from flask_restx import Resource
 
-from controllers.common.schema import register_response_schema_models, register_schema_models
+from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
 from controllers.console import console_ns
-from controllers.console.agent.app_helpers import resolve_agent_app_model
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import (
     RBACPermission,
@@ -28,9 +28,15 @@ from libs.login import login_required
 from models.model import App, AppMode
 from services.agent.composer_service import AgentComposerService
 from services.agent.composer_validator import ComposerConfigValidator
-from services.entities.agent_entities import ComposerSavePayload, WorkflowComposerCopyFromRosterPayload
+from services.entities.agent_entities import (
+    ComposerSavePayload,
+    WorkflowAgentComposerQuery,
+    WorkflowComposerCopyFromRosterPayload,
+)
 
-register_schema_models(console_ns, ComposerSavePayload, WorkflowComposerCopyFromRosterPayload)
+register_schema_models(
+    console_ns, ComposerSavePayload, WorkflowAgentComposerQuery, WorkflowComposerCopyFromRosterPayload
+)
 register_response_schema_models(
     console_ns,
     AgentAppComposerResponse,
@@ -41,27 +47,28 @@ register_response_schema_models(
 )
 
 
-def _resolve_agent_app_id(*, tenant_id: str, agent_id: UUID) -> str:
-    return resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id).id
-
-
 @console_ns.route("/apps/<uuid:app_id>/workflows/draft/nodes/<string:node_id>/agent-composer")
 class WorkflowAgentComposerApi(Resource):
     @console_ns.response(
         200, "Workflow agent composer state", console_ns.models[WorkflowAgentComposerResponse.__name__]
     )
+    @console_ns.doc(params=query_params_from_model(WorkflowAgentComposerQuery))
     @setup_required
     @login_required
     @account_initialization_required
     @get_app_model(mode=[AppMode.WORKFLOW, AppMode.ADVANCED_CHAT])
+    @with_current_user_id
     @with_current_tenant_id
-    def get(self, tenant_id: str, app_model: App, node_id: str):
+    def get(self, tenant_id: str, account_id: str, app_model: App, node_id: str):
+        query = WorkflowAgentComposerQuery.model_validate(request.args.to_dict(flat=True))
         return dump_response(
             WorkflowAgentComposerResponse,
             AgentComposerService.load_workflow_composer(
                 tenant_id=tenant_id,
                 app_id=app_model.id,
                 node_id=node_id,
+                account_id=account_id,
+                snapshot_id=query.snapshot_id,
             ),
         )
 
@@ -137,6 +144,7 @@ class WorkflowAgentComposerValidateApi(Resource):
     def post(self, tenant_id: str, app_model: App, node_id: str):
         payload = ComposerSavePayload.model_validate(console_ns.payload or {})
         ComposerConfigValidator.validate_publish_payload(payload)
+        AgentComposerService.validate_knowledge_datasets(tenant_id=tenant_id, agent_soul=payload.agent_soul)
         findings = AgentComposerService.collect_validation_findings(
             tenant_id=tenant_id,
             payload=payload,
@@ -228,10 +236,9 @@ class AgentComposerApi(Resource):
     @account_initialization_required
     @with_current_tenant_id
     def get(self, tenant_id: str, agent_id: UUID):
-        app_id = _resolve_agent_app_id(tenant_id=tenant_id, agent_id=agent_id)
         return dump_response(
             AgentAppComposerResponse,
-            AgentComposerService.load_agent_app_composer(tenant_id=tenant_id, app_id=app_id),
+            AgentComposerService.load_agent_composer(tenant_id=tenant_id, agent_id=str(agent_id)),
         )
 
     @console_ns.expect(console_ns.models[ComposerSavePayload.__name__])
@@ -244,13 +251,12 @@ class AgentComposerApi(Resource):
     @with_current_user_id
     @with_current_tenant_id
     def put(self, tenant_id: str, account_id: str, agent_id: UUID):
-        app_id = _resolve_agent_app_id(tenant_id=tenant_id, agent_id=agent_id)
         payload = ComposerSavePayload.model_validate(console_ns.payload or {})
         return dump_response(
             AgentAppComposerResponse,
-            AgentComposerService.save_agent_app_composer(
+            AgentComposerService.save_agent_composer(
                 tenant_id=tenant_id,
-                app_id=app_id,
+                agent_id=str(agent_id),
                 account_id=account_id,
                 payload=payload,
             ),
@@ -268,9 +274,10 @@ class AgentComposerValidateApi(Resource):
     @account_initialization_required
     @with_current_tenant_id
     def post(self, tenant_id: str, agent_id: UUID):
-        _resolve_agent_app_id(tenant_id=tenant_id, agent_id=agent_id)
+        AgentComposerService.load_agent_composer(tenant_id=tenant_id, agent_id=str(agent_id))
         payload = ComposerSavePayload.model_validate(console_ns.payload or {})
         ComposerConfigValidator.validate_publish_payload(payload)
+        AgentComposerService.validate_knowledge_datasets(tenant_id=tenant_id, agent_soul=payload.agent_soul)
         findings = AgentComposerService.collect_validation_findings(
             tenant_id=tenant_id,
             payload=payload,
@@ -290,12 +297,11 @@ class AgentComposerCandidatesApi(Resource):
     @with_current_user_id
     @with_current_tenant_id
     def get(self, tenant_id: str, current_user_id: str, agent_id: UUID):
-        app_id = _resolve_agent_app_id(tenant_id=tenant_id, agent_id=agent_id)
         return dump_response(
             AgentComposerCandidatesResponse,
             AgentComposerService.get_agent_app_candidates(
                 tenant_id=tenant_id,
-                app_id=app_id,
+                agent_id=str(agent_id),
                 user_id=current_user_id,
             ),
         )

api/controllers/console/agent/roster.py

@@ -7,7 +7,7 @@ from sqlalchemy import func, select
 
 from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
 from controllers.console import console_ns
-from controllers.console.agent.app_helpers import resolve_agent_app_model
+from controllers.console.agent.app_helpers import resolve_agent_app_model, resolve_agent_runtime_app_model
 from controllers.console.apikey import ApiKeyItem, ApiKeyList, BaseApiKeyListResource, BaseApiKeyResource
 from controllers.console.app.app import (
     AppDetailWithSite as GenericAppDetailWithSite,
@@ -54,8 +54,10 @@ from libs.datetime_utils import parse_time_range
 from libs.helper import dump_response
 from libs.login import login_required
 from models import Account
+from models.agent import Agent, AgentStatus
 from models.enums import ApiTokenType
 from models.model import ApiToken, App, IconType
+from services.agent.composer_service import AgentComposerService
 from services.agent.errors import AgentNotFoundError
 from services.agent.observability_service import (
     AgentLogQueryParams,
@@ -65,7 +67,7 @@ from services.agent.observability_service import (
 from services.agent.roster_service import AgentRosterService
 from services.app_service import AppListParams, AppService, CreateAppParams
 from services.enterprise.enterprise_service import EnterpriseService
-from services.entities.agent_entities import RosterListQuery
+from services.entities.agent_entities import ComposerSavePayload, RosterListQuery
 from services.feature_service import FeatureService
 
 
@@ -232,6 +234,8 @@ class AgentStatisticsQuery(BaseModel):
 
 class AgentAppPartial(GenericAppPartial):
     app_id: str | None = None
+    backing_app_id: str | None = None
+    hidden_app_backed: bool = False
     debug_conversation_id: str | None = None
     role: str | None = None
     active_config_is_published: bool = False
@@ -241,13 +245,49 @@ class AgentAppPartial(GenericAppPartial):
 
 class AgentAppDetailWithSite(GenericAppDetailWithSite):
     app_id: str | None = None
+    backing_app_id: str | None = None
+    hidden_app_backed: bool = False
     debug_conversation_id: str | None = None
+    debug_conversation_has_messages: bool = False
+    debug_conversation_message_count: int = 0
     role: str | None = None
     active_config_is_published: bool = False
 
 
 class AgentDebugConversationRefreshResponse(BaseModel):
     debug_conversation_id: str
+    debug_conversation_has_messages: bool = False
+    debug_conversation_message_count: int = 0
+
+
+class AgentPublishPayload(BaseModel):
+    version_note: str | None = Field(default=None, description="Optional note for this published Agent version")
+
+
+class AgentPublishResponse(BaseModel):
+    result: str
+    active_config_snapshot_id: str
+    active_config_snapshot: dict[str, object] | None = None
+    draft: dict[str, object] | None = None
+
+
+class AgentBuildDraftCheckoutPayload(BaseModel):
+    force: bool = Field(default=False, description="Overwrite the existing current-user build draft")
+
+
+class AgentBuildDraftResponse(BaseModel):
+    variant: str
+    draft: dict[str, object]
+    agent_soul: dict[str, object]
+
+
+class AgentBuildDraftApplyResponse(BaseModel):
+    result: str
+    draft: dict[str, object]
+
+
+class AgentSimpleResultResponse(BaseModel):
+    result: str
 
 
 class AgentAppPagination(GenericAppPagination):
@@ -261,6 +301,9 @@ register_schema_models(
     AgentAppCreatePayload,
     AgentAppUpdatePayload,
     AgentAppCopyPayload,
+    AgentPublishPayload,
+    AgentBuildDraftCheckoutPayload,
+    ComposerSavePayload,
     AgentApiStatusPayload,
     AgentInviteOptionsQuery,
     AgentLogsQuery,
@@ -277,6 +320,10 @@ register_response_schema_models(
     AgentAppDetailWithSite,
     AgentAppPartial,
     AgentDebugConversationRefreshResponse,
+    AgentPublishResponse,
+    AgentBuildDraftResponse,
+    AgentBuildDraftApplyResponse,
+    AgentSimpleResultResponse,
     AgentConfigSnapshotDetailResponse,
     AgentConfigSnapshotListResponse,
     AgentConfigSnapshotRestoreResponse,
@@ -294,7 +341,7 @@ def _agent_roster_service() -> AgentRosterService:
     return AgentRosterService(db.session)
 
 
-def _serialize_agent_app_detail(app_model, *, current_user: Account) -> dict:
+def _serialize_agent_app_detail(app_model, *, current_user: Account, agent_id: str | None = None) -> dict:
     """Serialize an Agent App detail using roster-only DTOs.
 
     `/agent` responses are roster-shaped rather than raw app-shaped: `id`
@@ -311,17 +358,35 @@ def _serialize_agent_app_detail(app_model, *, current_user: Account) -> dict:
 
     roster_service = _agent_roster_service()
     payload = AgentAppDetailWithSite.model_validate(app_model, from_attributes=True).model_dump(mode="json")
-    agent = roster_service.get_app_backing_agent(tenant_id=app_model.tenant_id, app_id=str(app_model.id))
+    agent = (
+        db.session.scalar(
+            select(Agent).where(
+                Agent.tenant_id == app_model.tenant_id,
+                Agent.id == agent_id,
+                Agent.status == AgentStatus.ACTIVE,
+            )
+        )
+        if agent_id
+        else roster_service.get_app_backing_agent(tenant_id=app_model.tenant_id, app_id=str(app_model.id))
+    )
     if not agent:
         raise AgentNotFoundError()
     payload.pop("bound_agent_id", None)
-    payload["app_id"] = str(app_model.id)
+    payload["app_id"] = agent.app_id
+    payload["backing_app_id"] = roster_service.runtime_backing_app_id(agent)
+    payload["hidden_app_backed"] = bool(agent.backing_app_id and agent.backing_app_id != agent.app_id)
     payload["id"] = agent.id
-    payload["debug_conversation_id"] = roster_service.get_or_create_agent_app_debug_conversation_id(
+    debug_conversation_id = roster_service.get_or_create_agent_app_debug_conversation_id(
         tenant_id=app_model.tenant_id,
         agent_id=agent.id,
         account_id=current_user.id,
     )
+    message_count = roster_service.count_agent_app_debug_conversation_messages(
+        conversation_id=debug_conversation_id,
+    )
+    payload["debug_conversation_id"] = debug_conversation_id
+    payload["debug_conversation_has_messages"] = message_count > 0
+    payload["debug_conversation_message_count"] = message_count
     payload["role"] = agent.role or ""
     payload["active_config_is_published"] = roster_service.active_config_is_published(
         tenant_id=app_model.tenant_id,
@@ -365,6 +430,8 @@ def _serialize_agent_app_pagination(app_pagination, *, tenant_id: str, current_u
         agent = agents_by_app_id.get(app_id)
         if agent:
             item["app_id"] = app_id
+            item["backing_app_id"] = agent.backing_app_id or app_id
+            item["hidden_app_backed"] = False
             item["id"] = agent.id
             item["debug_conversation_id"] = debug_conversation_ids_by_agent_id.get(agent.id)
             item["role"] = agent.role or ""
@@ -516,8 +583,8 @@ class AgentAppApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def get(self, tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = _resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
-        return _serialize_agent_app_detail(app_model, current_user=current_user)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        return _serialize_agent_app_detail(app_model, current_user=current_user, agent_id=str(agent_id))
 
     @console_ns.expect(console_ns.models[AgentAppUpdatePayload.__name__])
     @console_ns.response(200, "Agent app updated successfully", console_ns.models[AgentAppDetailWithSite.__name__])
@@ -578,8 +645,116 @@ class AgentDebugConversationRefreshApi(Resource):
             agent_id=str(agent_id),
             account_id=current_user.id,
         )
-        return AgentDebugConversationRefreshResponse(debug_conversation_id=debug_conversation_id).model_dump(
-            mode="json"
+        return AgentDebugConversationRefreshResponse(
+            debug_conversation_id=debug_conversation_id,
+            debug_conversation_has_messages=False,
+            debug_conversation_message_count=0,
+        ).model_dump(mode="json")
+
+
+@console_ns.route("/agent/<uuid:agent_id>/publish")
+class AgentPublishApi(Resource):
+    @console_ns.expect(console_ns.models[AgentPublishPayload.__name__])
+    @console_ns.response(200, "Agent draft published", console_ns.models[AgentPublishResponse.__name__])
+    @console_ns.response(403, "Insufficient permissions")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @with_current_user
+    @with_current_tenant_id
+    def post(self, tenant_id: str, current_user: Account, agent_id: UUID):
+        args = AgentPublishPayload.model_validate(console_ns.payload or {})
+        return AgentComposerService.publish_agent_app_draft(
+            tenant_id=tenant_id,
+            agent_id=str(agent_id),
+            account_id=current_user.id,
+            version_note=args.version_note,
+        )
+
+
+@console_ns.route("/agent/<uuid:agent_id>/build-draft/checkout")
+class AgentBuildDraftCheckoutApi(Resource):
+    @console_ns.expect(console_ns.models[AgentBuildDraftCheckoutPayload.__name__])
+    @console_ns.response(200, "Agent build draft checked out", console_ns.models[AgentBuildDraftResponse.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @with_current_user
+    @with_current_tenant_id
+    def post(self, tenant_id: str, current_user: Account, agent_id: UUID):
+        args = AgentBuildDraftCheckoutPayload.model_validate(console_ns.payload or {})
+        return AgentComposerService.checkout_agent_app_build_draft(
+            tenant_id=tenant_id,
+            agent_id=str(agent_id),
+            account_id=current_user.id,
+            force=args.force,
+        )
+
+
+@console_ns.route("/agent/<uuid:agent_id>/build-draft")
+class AgentBuildDraftApi(Resource):
+    @console_ns.response(200, "Agent build draft", console_ns.models[AgentBuildDraftResponse.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @with_current_user
+    @with_current_tenant_id
+    def get(self, tenant_id: str, current_user: Account, agent_id: UUID):
+        return AgentComposerService.load_agent_app_build_draft(
+            tenant_id=tenant_id,
+            agent_id=str(agent_id),
+            account_id=current_user.id,
+        )
+
+    @console_ns.expect(console_ns.models[ComposerSavePayload.__name__])
+    @console_ns.response(200, "Agent build draft saved", console_ns.models[AgentBuildDraftResponse.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @with_current_user
+    @with_current_tenant_id
+    def put(self, tenant_id: str, current_user: Account, agent_id: UUID):
+        payload = ComposerSavePayload.model_validate(console_ns.payload or {})
+        return AgentComposerService.save_agent_app_build_draft(
+            tenant_id=tenant_id,
+            agent_id=str(agent_id),
+            account_id=current_user.id,
+            payload=payload,
+        )
+
+    @console_ns.response(200, "Agent build draft discarded", console_ns.models[AgentSimpleResultResponse.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @with_current_user
+    @with_current_tenant_id
+    def delete(self, tenant_id: str, current_user: Account, agent_id: UUID):
+        return AgentComposerService.discard_agent_app_build_draft(
+            tenant_id=tenant_id,
+            agent_id=str(agent_id),
+            account_id=current_user.id,
+        )
+
+
+@console_ns.route("/agent/<uuid:agent_id>/build-draft/apply")
+class AgentBuildDraftApplyApi(Resource):
+    @console_ns.response(200, "Agent build draft applied", console_ns.models[AgentBuildDraftApplyResponse.__name__])
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @with_current_user
+    @with_current_tenant_id
+    def post(self, tenant_id: str, current_user: Account, agent_id: UUID):
+        return AgentComposerService.apply_agent_app_build_draft(
+            tenant_id=tenant_id,
+            agent_id=str(agent_id),
+            account_id=current_user.id,
         )
 
 
@@ -712,7 +887,7 @@ class AgentLogsApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def get(self, tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = _resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         query_data: dict[str, object] = dict(request.args.to_dict(flat=True))
         query_data["sources"] = _multi_query_values("sources", "source")
         query_data["statuses"] = _multi_query_values("statuses", "status")
@@ -749,7 +924,7 @@ class AgentLogMessagesApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def get(self, tenant_id: str, current_user: Account, agent_id: UUID, conversation_id: UUID):
-        app_model = _resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         query_data: dict[str, object] = dict(request.args.to_dict(flat=True))
         query_data["sources"] = _multi_query_values("sources", "source")
         query_data["statuses"] = _multi_query_values("statuses", "status")
@@ -786,7 +961,7 @@ class AgentLogSourcesApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def get(self, tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = _resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         payload = _agent_observability_service().list_log_sources(app=app_model, agent_id=str(agent_id))
         return dump_response(AgentLogSourceListResponse, payload)
 
@@ -805,7 +980,7 @@ class AgentStatisticsSummaryApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def get(self, tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = _resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         query = AgentStatisticsQuery.model_validate(request.args.to_dict(flat=True))
         timezone = current_user.timezone or "UTC"
         start, end = _parse_observability_time_range(query.start, query.end, current_user)

api/controllers/console/app/agent.py

@@ -13,7 +13,7 @@ from controllers.common.schema import (
     register_schema_models,
 )
 from controllers.console import console_ns
-from controllers.console.agent.app_helpers import resolve_agent_app_model
+from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import (
     RBACPermission,
@@ -351,7 +351,7 @@ class AgentSkillUploadByAgentApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def post(self, tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         return _upload_skill_for_app(current_user=current_user, app_model=app_model)
 
 
@@ -394,7 +394,7 @@ class AgentDriveFilesByAgentApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def post(self, tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         return _commit_drive_file_for_app(current_user=current_user, app_model=app_model, allow_node_id=False)
 
     @console_ns.doc("delete_agent_drive_file_by_agent")
@@ -407,7 +407,7 @@ class AgentDriveFilesByAgentApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def delete(self, tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         return _delete_drive_file_for_app(current_user=current_user, app_model=app_model, allow_node_id=False)
 
 
@@ -454,7 +454,7 @@ class AgentSkillByAgentApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def delete(self, tenant_id: str, current_user: Account, agent_id: UUID, slug: str):
-        app_model = resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         return _delete_skill_for_app(current_user=current_user, app_model=app_model, slug=slug, allow_node_id=False)
 
 
@@ -494,7 +494,7 @@ class AgentSkillInferToolsByAgentApi(Resource):
     @account_initialization_required
     @with_current_tenant_id
     def post(self, tenant_id: str, agent_id: UUID, slug: str):
-        app_model = resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         return _infer_skill_tools_for_app(app_model=app_model, slug=slug)
 
 

api/controllers/console/app/agent_app_feature.py

@@ -17,7 +17,7 @@ from pydantic import BaseModel, Field
 from controllers.common.fields import SimpleResultResponse
 from controllers.common.schema import register_response_schema_models, register_schema_models
 from controllers.console import console_ns
-from controllers.console.agent.app_helpers import resolve_agent_app_model
+from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model
 from controllers.console.wraps import (
     RBACPermission,
     RBACResourceScope,
@@ -87,7 +87,7 @@ class AgentAppFeatureConfigResource(Resource):
     @with_current_user
     @with_current_tenant_id
     def post(self, tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         args = AgentAppFeaturesPayload.model_validate(console_ns.payload or {})
 
         new_app_model_config = AgentAppFeatureConfigService.update_features(

api/controllers/console/app/agent_app_sandbox.py

@@ -22,7 +22,7 @@ from controllers.common.schema import (
     register_schema_models,
 )
 from controllers.console import console_ns
-from controllers.console.agent.app_helpers import resolve_agent_app_model
+from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required, with_current_tenant_id
 from fields.base import ResponseModel
@@ -144,7 +144,7 @@ class AgentAppSandboxListResource(Resource):
     @account_initialization_required
     @with_current_tenant_id
     def get(self, tenant_id: str, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         query = query_params_from_request(AgentSandboxListQuery)
         try:
             result = AgentAppSandboxService().list_files(
@@ -169,7 +169,7 @@ class AgentAppSandboxReadResource(Resource):
     @account_initialization_required
     @with_current_tenant_id
     def get(self, tenant_id: str, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         query = query_params_from_request(AgentSandboxFileQuery)
         try:
             result = AgentAppSandboxService().read_file(
@@ -194,7 +194,7 @@ class AgentAppSandboxUploadResource(Resource):
     @account_initialization_required
     @with_current_tenant_id
     def post(self, tenant_id: str, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         payload = AgentSandboxUploadPayload.model_validate(request.get_json(silent=True) or {})
         try:
             result = AgentAppSandboxService().upload_file(

api/controllers/console/app/agent_drive_inspector.py

@@ -25,7 +25,7 @@ from controllers.common.schema import (
     register_response_schema_models,
 )
 from controllers.console import console_ns
-from controllers.console.agent.app_helpers import resolve_agent_app_model
+from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model
 from controllers.console.app.wraps import get_app_model
 from controllers.console.wraps import account_initialization_required, setup_required, with_current_tenant_id
 from fields.base import ResponseModel
@@ -182,7 +182,7 @@ class AgentDriveListByAgentApi(Resource):
     @with_current_tenant_id
     def get(self, tenant_id: str, agent_id: UUID):
         query = query_params_from_request(AgentDriveListByAgentQuery)
-        resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         try:
             items = AgentDriveService().manifest(tenant_id=tenant_id, agent_id=str(agent_id), prefix=query.prefix)
         except AgentDriveError as exc:
@@ -201,7 +201,7 @@ class AgentDriveSkillListByAgentApi(Resource):
     @account_initialization_required
     @with_current_tenant_id
     def get(self, tenant_id: str, agent_id: UUID):
-        resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         try:
             items = AgentDriveService().list_skills(tenant_id=tenant_id, agent_id=str(agent_id))
         except AgentDriveError as exc:
@@ -220,7 +220,7 @@ class AgentDriveSkillInspectByAgentApi(Resource):
     @account_initialization_required
     @with_current_tenant_id
     def get(self, tenant_id: str, agent_id: UUID, skill_path: str):
-        resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         try:
             return _json_response(
                 AgentDriveService().inspect_skill(
@@ -245,7 +245,7 @@ class AgentDrivePreviewByAgentApi(Resource):
     @with_current_tenant_id
     def get(self, tenant_id: str, agent_id: UUID):
         query = query_params_from_request(AgentDriveFileByAgentQuery)
-        resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         try:
             return AgentDriveService().preview(tenant_id=tenant_id, agent_id=str(agent_id), key=query.key)
         except AgentDriveError as exc:
@@ -264,7 +264,7 @@ class AgentDriveDownloadByAgentApi(Resource):
     @with_current_tenant_id
     def get(self, tenant_id: str, agent_id: UUID):
         query = query_params_from_request(AgentDriveFileByAgentQuery)
-        resolve_agent_app_model(tenant_id=tenant_id, agent_id=agent_id)
+        resolve_agent_runtime_app_model(tenant_id=tenant_id, agent_id=agent_id)
         try:
             url = AgentDriveService().download_url(tenant_id=tenant_id, agent_id=str(agent_id), key=query.key)
         except AgentDriveError as exc:

api/controllers/console/app/annotation.py

@@ -4,6 +4,8 @@ from uuid import UUID
 from flask import abort, make_response, request
 from flask_restx import Resource
 from pydantic import BaseModel, Field, TypeAdapter, field_validator
+from sqlalchemy import select
+from werkzeug.exceptions import NotFound
 
 from controllers.common.errors import NoFileUploadedError, TooManyFilesError
 from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
@@ -30,7 +32,8 @@ from fields.annotation_fields import (
 )
 from fields.base import ResponseModel
 from libs.helper import uuid_value
-from libs.login import login_required
+from libs.login import current_account_with_tenant, login_required
+from models.model import App
 from services.annotation_service import (
     AppAnnotationService,
     EnableAnnotationArgs,
@@ -38,6 +41,17 @@ from services.annotation_service import (
     UpdateAnnotationSettingArgs,
     UpsertAnnotationArgs,
 )
+from services.app_ref_service import AppRef, AppRefService
+
+
+def _get_app_ref(app_id: str) -> AppRef:
+    _, current_tenant_id = current_account_with_tenant()
+    app = db.session.scalar(
+        select(App).where(App.id == app_id, App.tenant_id == current_tenant_id, App.status == "normal").limit(1)
+    )
+    if app is None:
+        raise NotFound("App not found")
+    return AppRefService.create_app_ref(app)
 
 
 class AnnotationReplyPayload(BaseModel):
@@ -330,7 +344,8 @@ class AnnotationApi(Resource):
                     "message": "annotation_ids are required if the parameter is provided.",
                 }, 400
 
-            AppAnnotationService.delete_app_annotations_in_batch(str(app_id), annotation_ids)
+            app_ref = _get_app_ref(str(app_id))
+            AppAnnotationService.delete_app_annotations_in_batch(app_ref, annotation_ids)
             return "", 204
         # If no annotation_ids are provided, handle clearing all annotations
         else:
@@ -389,9 +404,9 @@ class AnnotationUpdateDeleteApi(Resource):
             update_args["answer"] = args.answer
         if args.question is not None:
             update_args["question"] = args.question
-        annotation = AppAnnotationService.update_app_annotation_directly(
-            update_args, str(app_id), str(annotation_id), db.session
-        )
+        app_ref = _get_app_ref(str(app_id))
+        annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
+        annotation = AppAnnotationService.update_app_annotation_directly(update_args, annotation_ref, db.session)
         return Annotation.model_validate(annotation, from_attributes=True).model_dump(mode="json")
 
     @setup_required
@@ -401,7 +416,9 @@ class AnnotationUpdateDeleteApi(Resource):
     @rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_EDIT)
     @console_ns.response(204, "Annotation deleted successfully")
     def delete(self, app_id: UUID, annotation_id: UUID):
-        AppAnnotationService.delete_app_annotation(str(app_id), str(annotation_id), db.session)
+        app_ref = _get_app_ref(str(app_id))
+        annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
+        AppAnnotationService.delete_app_annotation(annotation_ref, db.session)
         return "", 204
 
 
@@ -514,8 +531,12 @@ class AnnotationHitHistoryListApi(Resource):
     def get(self, app_id: UUID, annotation_id: UUID):
         page = request.args.get("page", default=1, type=int)
         limit = request.args.get("limit", default=20, type=int)
+        app_ref = _get_app_ref(str(app_id))
+        annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
         annotation_hit_history_list, total = AppAnnotationService.get_annotation_hit_histories(
-            str(app_id), str(annotation_id), page, limit
+            annotation_ref,
+            page,
+            limit,
         )
         history_models = TypeAdapter(list[AnnotationHitHistory]).validate_python(
             annotation_hit_history_list, from_attributes=True

api/controllers/console/app/app.py

@@ -331,7 +331,7 @@ class ModelConfig(ResponseModel):
         return to_timestamp(value)
 
 
-class Site(ResponseModel):
+class AppDetailSiteResponse(ResponseModel):
     access_token: str | None = Field(default=None, validation_alias="code")
     code: str | None = None
     title: str | None = None
@@ -345,6 +345,7 @@ class Site(ResponseModel):
     customize_domain: str | None = None
     copyright: str | None = None
     privacy_policy: str | None = None
+    input_placeholder: str | None = None
     custom_disclaimer: str | None = None
     customize_token_strategy: str | None = None
     prompt_public: bool | None = None
@@ -461,7 +462,7 @@ class AppDetailWithSite(AppDetail):
     api_base_url: str | None = None
     max_active_requests: int | None = None
     deleted_tools: list[DeletedTool] = Field(default_factory=list)
-    site: Site | None = None
+    site: AppDetailSiteResponse | None = None
     # For Agent App type: the roster Agent backing this app (None otherwise).
     bound_agent_id: str | None = None
     # For Agent App responses exposed through /agent.
@@ -546,7 +547,7 @@ register_schema_models(
     WorkflowPartial,
     ModelConfigPartial,
     ModelConfig,
-    Site,
+    AppDetailSiteResponse,
     DeletedTool,
     AppDetail,
     AppExportResponse,

api/controllers/console/app/audio.py

@@ -32,8 +32,9 @@ from controllers.console.wraps import (
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from extensions.ext_database import db
 from graphon.model_runtime.errors.invoke import InvokeError
-from libs.login import login_required
+from libs.login import current_user, login_required
 from models import App, AppMode
+from services.app_ref_service import AppRefService
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,
@@ -140,13 +141,21 @@ class ChatMessageTextApi(Resource):
     def post(self, app_model: App):
         try:
             payload = TextToSpeechPayload.model_validate(console_ns.payload)
+            message_ref = None
+            if payload.message_id:
+                app_ref = AppRefService.create_app_ref(app_model)
+                message_ref = AppRefService.create_message_ref(
+                    app_ref,
+                    payload.message_id,
+                    account_id=current_user.id,
+                )
 
             response = AudioService.transcript_tts(
                 app_model=app_model,
                 session=db.session,
                 text=payload.text,
                 voice=payload.voice,
-                message_id=payload.message_id,
+                message_ref=message_ref,
                 is_draft=True,
             )
             return response

api/controllers/console/app/completion.py

@@ -1,4 +1,6 @@
+import json
 import logging
+from collections.abc import Generator
 from typing import Any, Literal
 from uuid import UUID
 
@@ -11,7 +13,7 @@ import services
 from controllers.common.fields import GeneratedAppResponse, SimpleResultResponse
 from controllers.common.schema import register_response_schema_models, register_schema_models
 from controllers.console import console_ns
-from controllers.console.agent.app_helpers import resolve_agent_app_model
+from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model
 from controllers.console.app.error import (
     AppUnavailableError,
     CompletionRequestError,
@@ -34,6 +36,7 @@ from controllers.console.wraps import (
 )
 from controllers.web.error import InvokeRateLimitError as InvokeRateLimitHttpError
 from core.app.entities.app_invoke_entities import InvokeFrom
+from core.app.features.rate_limiting.rate_limit import RateLimitGenerator
 from core.errors.error import (
     ModelCurrentlyNotSupportError,
     ProviderTokenNotInitError,
@@ -93,6 +96,10 @@ class ChatMessagePayload(BaseMessagePayload):
     query: str = Field(..., description="User query")
     conversation_id: str | None = Field(default=None, description="Conversation ID")
     parent_message_id: str | None = Field(default=None, description="Parent message ID")
+    draft_type: Literal["draft", "debug_build"] = Field(
+        default="draft",
+        description="Agent App debug config source. Use debug_build while the Agent is in build mode.",
+    )
 
     @field_validator("conversation_id", "parent_message_id")
     @classmethod
@@ -102,6 +109,32 @@ class ChatMessagePayload(BaseMessagePayload):
         return uuid_value(value)
 
 
+_BUILD_CHAT_FINALIZATION_QUERY = """Finalize this Build chat configuration for the agent.
+
+This step is only for persisting Agent config changes discovered in the current Build chat. Do not install packages,
+edit workspace files, run validation or debugging commands, make exploratory checks, or perform other work.
+
+Use only the current Build chat message history to identify changes that need to be persisted. Do not inspect, test, or
+validate old config unless the message history already shows that the old config is invalid.
+
+Persist only the build-draft config resources that need to change, using the Agent config CLI usage provided in the
+runtime prompt:
+
+- config files for reusable artifacts that should be available later,
+- config skills for reusable procedures or tools that should be available later,
+- config env when environment keys or values need to be recorded,
+- config note for concise durable context when useful.
+
+When updating the config note, record only durable context needed by later runs, such as:
+
+- what you installed or configured outside the workspace for this agent,
+- where those external updates live, including CLI tools, packages, and persistent $HOME paths,
+- how the agent should use it in later runs,
+- any setup, authentication, or user action still required.
+
+After config persistence completes, respond FINISHED."""
+
+
 register_schema_models(console_ns, CompletionMessagePayload, ChatMessagePayload)
 register_response_schema_models(console_ns, GeneratedAppResponse, SimpleResultResponse)
 
@@ -218,7 +251,7 @@ class AgentChatMessageApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def post(self, current_tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
         return _create_chat_message(
             current_tenant_id=current_tenant_id,
             current_user=current_user,
@@ -227,6 +260,31 @@ class AgentChatMessageApi(Resource):
         )
 
 
+@console_ns.route("/agent/<uuid:agent_id>/build-chat/finalize")
+class AgentBuildChatFinalizeApi(Resource):
+    @console_ns.doc("finalize_agent_build_chat")
+    @console_ns.doc(description="Run a build-draft Agent App turn that asks the agent to push config updates")
+    @console_ns.doc(params={"agent_id": "Agent ID"})
+    @console_ns.response(200, "Success", console_ns.models[SimpleResultResponse.__name__])
+    @console_ns.response(400, "Invalid request parameters")
+    @console_ns.response(404, "Agent, build draft, or conversation not found")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @edit_permission_required
+    @rbac_permission_required(RBACResourceScope.APP, RBACPermission.APP_TEST_AND_RUN)
+    @with_current_user
+    @with_current_tenant_id
+    def post(self, current_tenant_id: str, current_user: Account, agent_id: UUID):
+        app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
+        return _create_build_chat_finalization_message(
+            current_tenant_id=current_tenant_id,
+            current_user=current_user,
+            app_model=app_model,
+            agent_id=str(agent_id),
+        )
+
+
 @console_ns.route("/apps/<uuid:app_id>/chat-messages/<string:task_id>/stop")
 class ChatMessageStopApi(Resource):
     @console_ns.doc("stop_chat_message")
@@ -254,7 +312,7 @@ class AgentChatMessageStopApi(Resource):
     @with_current_user_id
     @with_current_tenant_id
     def post(self, current_tenant_id: str, current_user_id: str, agent_id: UUID, task_id: str):
-        app_model = resolve_agent_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
         return _stop_chat_message(current_user_id=current_user_id, app_model=app_model, task_id=task_id)
 
 
@@ -280,7 +338,11 @@ def _resolve_current_user_agent_debug_conversation_id(
 
 
 def _create_chat_message(
-    *, current_user: Account, app_model: App, current_tenant_id: str | None = None, agent_id: str | None = None
+    *,
+    current_user: Account,
+    app_model: App,
+    current_tenant_id: str | None = None,
+    agent_id: str | None = None,
 ):
     raw_payload = console_ns.payload or {}
     args_model = ChatMessagePayload.model_validate(raw_payload)
@@ -310,12 +372,104 @@ def _create_chat_message(
     if external_trace_id:
         args["external_trace_id"] = external_trace_id
 
-    try:
-        response = AppGenerateService.generate(
-            app_model=app_model, user=current_user, args=args, invoke_from=InvokeFrom.DEBUGGER, streaming=streaming
-        )
+    return _generate_chat_message_response(
+        current_user=current_user,
+        app_model=app_model,
+        args=args,
+        streaming=streaming,
+    )
 
-        return helper.compact_generate_response(response)
+
+def _create_build_chat_finalization_message(
+    *, current_user: Account, app_model: App, current_tenant_id: str, agent_id: str
+):
+    debug_conversation_id = _resolve_current_user_agent_debug_conversation_id(
+        current_tenant_id=current_tenant_id,
+        current_user=current_user,
+        app_model=app_model,
+        agent_id=agent_id,
+    )
+    args: dict[str, Any] = {
+        "query": _BUILD_CHAT_FINALIZATION_QUERY,
+        "inputs": {},
+        "response_mode": "streaming",
+        "draft_type": "debug_build",
+        "conversation_id": debug_conversation_id,
+        "auto_generate_name": False,
+    }
+    external_trace_id = get_external_trace_id(request)
+    if external_trace_id:
+        args["external_trace_id"] = external_trace_id
+
+    response = _generate_chat_message(
+        current_user=current_user,
+        app_model=app_model,
+        args=args,
+        streaming=True,
+    )
+    _drain_streaming_generate_response(response)
+    return {"result": "success"}, 200
+
+
+def _drain_streaming_generate_response(response: RateLimitGenerator | Generator[str, None, None]) -> None:
+    """Consume a streamed app-generate response until a terminal message event arrives.
+
+    Finalize keeps the normal Agent App streaming path so the existing queue,
+    persistence, and runtime-session behavior stay intact. The console API only
+    changes the HTTP boundary: it drains the SSE stream server-side and returns
+    success after the generated build-chat message reaches ``message_end``.
+    """
+    close = getattr(response, "close", None)
+    try:
+        for chunk in response:
+            for raw_event in chunk.split("\n\n"):
+                if not raw_event.strip():
+                    continue
+
+                event_name: str | None = None
+                data_lines: list[str] = []
+                for line in raw_event.splitlines():
+                    if line.startswith("event: "):
+                        event_name = line.removeprefix("event: ").strip()
+                    elif line.startswith("data: "):
+                        data_lines.append(line.removeprefix("data: "))
+
+                if not data_lines:
+                    if event_name == "ping":
+                        continue
+                    continue
+
+                payload = json.loads("\n".join(data_lines))
+                if not isinstance(payload, dict):
+                    continue
+
+                payload_event = payload.get("event")
+                if payload_event == "message_end":
+                    return
+                if payload_event == "error":
+                    raise CompletionRequestError(str(payload.get("message") or "Build chat finalization failed."))
+    finally:
+        if callable(close):
+            close()
+
+    raise CompletionRequestError("Build chat finalization did not complete.")
+
+
+def _generate_chat_message(
+    *,
+    current_user: Account,
+    app_model: App,
+    args: dict[str, Any],
+    streaming: bool,
+):
+    try:
+        return AppGenerateService.generate(
+            app_model=app_model,
+            user=current_user,
+            args=args,
+            invoke_from=InvokeFrom.DEBUGGER,
+            streaming=streaming,
+        )
     except services.errors.conversation.ConversationNotExistsError:
         raise NotFound("Conversation Not Exists.")
     except services.errors.conversation.ConversationCompletedError:
@@ -331,6 +485,8 @@ def _create_chat_message(
         raise ProviderModelCurrentlyNotSupportError()
     except InvokeRateLimitError as ex:
         raise InvokeRateLimitHttpError(ex.description)
+    except CompletionRequestError:
+        raise
     except InvokeError as e:
         raise CompletionRequestError(e.description)
     except ValueError as e:
@@ -340,6 +496,22 @@ def _create_chat_message(
         raise InternalServerError()
 
 
+def _generate_chat_message_response(
+    *,
+    current_user: Account,
+    app_model: App,
+    args: dict[str, Any],
+    streaming: bool,
+):
+    response = _generate_chat_message(
+        current_user=current_user,
+        app_model=app_model,
+        args=args,
+        streaming=streaming,
+    )
+    return helper.compact_generate_response(response)
+
+
 def _stop_chat_message(*, current_user_id: str, app_model: App, task_id: str):
     AppTaskService.stop_task(
         task_id=task_id,

api/controllers/console/app/generator.py

@@ -1,8 +1,10 @@
-from collections.abc import Sequence
+import json
+from collections.abc import Generator, Sequence
 from typing import Any, Literal
 
 from flask_restx import Resource
 from pydantic import BaseModel, Field, RootModel
+from sqlalchemy import select
 from sqlalchemy.orm import Session
 
 from controllers.common.fields import SimpleDataResponse
@@ -23,8 +25,10 @@ from core.helper.code_executor.javascript.javascript_code_provider import Javasc
 from core.helper.code_executor.python3.python3_code_provider import Python3CodeProvider
 from core.llm_generator.entities import RuleCodeGeneratePayload, RuleGeneratePayload, RuleStructuredOutputPayload
 from core.llm_generator.llm_generator import LLMGenerator
+from core.workflow.generator.types import WorkflowGenerateErrorCode
 from graphon.model_runtime.entities.llm_entities import LLMMode
 from graphon.model_runtime.errors.invoke import InvokeError
+from libs.helper import compact_generate_response
 from libs.login import login_required
 from models import App
 from services.workflow_generator_service import WorkflowGeneratorService
@@ -64,7 +68,10 @@ class WorkflowGeneratePayload(BaseModel):
     can reuse its existing handler.
     """
 
-    mode: Literal["workflow", "advanced-chat"] = Field(..., description="Target app mode for the generated graph")
+    mode: Literal["workflow", "advanced-chat", "auto"] = Field(
+        ...,
+        description="Target app mode for the generated graph; 'auto' lets the backend classify the instruction",
+    )
     instruction: str = Field(..., description="Natural-language workflow description")
     ideal_output: str = Field(default="", description="Optional sample output for grounding")
     model_config_data: ModelConfig = Field(
@@ -78,6 +85,19 @@ class WorkflowGeneratePayload(BaseModel):
     )
 
 
+class WorkflowInstructionSuggestionsPayload(BaseModel):
+    """Payload for the workflow-generator instruction-suggestions endpoint.
+
+    Runs before the user picks a model, so the suggestions come from the
+    tenant's default model. The underlying generator never raises — an empty
+    ``suggestions`` list is a valid 200 (soft-fail).
+    """
+
+    mode: Literal["workflow", "advanced-chat"] = Field(..., description="Target app mode for the suggestions")
+    language: str | None = Field(default=None, description="Optional language to write the suggestions in")
+    count: int = Field(default=4, ge=1, le=6, description="Number of suggestions to return (1-6)")
+
+
 class GeneratorResponse(RootModel[Any]):
     root: Any
 
@@ -91,6 +111,7 @@ register_schema_models(
     InstructionGeneratePayload,
     InstructionTemplatePayload,
     WorkflowGeneratePayload,
+    WorkflowInstructionSuggestionsPayload,
     ModelConfig,
 )
 register_response_schema_models(console_ns, GeneratorResponse, SimpleDataResponse)
@@ -216,7 +237,9 @@ class InstructionGenerateApi(Resource):
         try:
             # Generate from nothing for a workflow node
             if (args.current in (code_template, "")) and args.node_id != "":
-                app = session.get(App, args.flow_id)
+                app = session.scalar(
+                    select(App).where(App.id == args.flow_id, App.tenant_id == current_tenant_id).limit(1)
+                )
                 if not app:
                     return {"error": f"app {args.flow_id} not found"}, 400
                 workflow = WorkflowService().get_draft_workflow(app_model=app, session=session)
@@ -313,6 +336,34 @@ class InstructionGenerationTemplateApi(Resource):
                 raise ValueError(f"Invalid type: {args.type}")
 
 
+def _workflow_instruction_guard(args: WorkflowGeneratePayload) -> tuple[dict, int] | None:
+    """Shared boundary guard for the workflow-generate endpoints.
+
+    Returns a ``(body, 400)`` tuple when the instruction is empty / whitespace
+    or either free-text field exceeds the cap, else ``None``. Pydantic only
+    validates the field is a str; a whitespace-only or pasted-document input
+    would otherwise waste a slow planner+builder roundtrip on a response the
+    validator rejects anyway. Both the blocking and streaming endpoints call
+    this so they reject identical inputs.
+    """
+    if not args.instruction.strip():
+        return {
+            "error": "Instruction is required",
+            "errors": [{"code": WorkflowGenerateErrorCode.EMPTY_INSTRUCTION, "detail": "Instruction is required"}],
+        }, 400
+    if len(args.instruction) > _MAX_INSTRUCTION_LENGTH or len(args.ideal_output) > _MAX_INSTRUCTION_LENGTH:
+        return {
+            "error": "Instruction is too long",
+            "errors": [
+                {
+                    "code": WorkflowGenerateErrorCode.INSTRUCTION_TOO_LONG,
+                    "detail": f"Instruction and ideal output must each be at most {_MAX_INSTRUCTION_LENGTH} characters",
+                }
+            ],
+        }, 400
+    return None
+
+
 @console_ns.route("/workflow-generate")
 class WorkflowGenerateApi(Resource):
     """Generate a Workflow / Chatflow draft graph from a natural-language description.
@@ -335,31 +386,11 @@ class WorkflowGenerateApi(Resource):
     def post(self, current_tenant_id: str):
         args = WorkflowGeneratePayload.model_validate(console_ns.payload)
 
-        # Reject obviously-empty instructions at the boundary — Pydantic only
-        # validates ``instruction`` is a str, but a whitespace-only string
-        # would still hit the LLM and waste a planner+builder roundtrip on a
-        # response that the postprocess validator would reject anyway.
-        if not args.instruction.strip():
-            return {
-                "error": "Instruction is required",
-                "errors": [{"code": "EMPTY_INSTRUCTION", "detail": "Instruction is required"}],
-            }, 400
-
-        # Bound the prompt at the boundary too: an arbitrarily long
-        # instruction (or pasted document) blows the planner/builder context
-        # window and fails with an opaque provider error after two slow LLM
-        # calls. The cap matches the frontend textarea's maxLength.
-        if len(args.instruction) > _MAX_INSTRUCTION_LENGTH or len(args.ideal_output) > _MAX_INSTRUCTION_LENGTH:
-            return {
-                "error": "Instruction is too long",
-                "errors": [
-                    {
-                        "code": "INSTRUCTION_TOO_LONG",
-                        "detail": f"Instruction and ideal output must each be at most "
-                        f"{_MAX_INSTRUCTION_LENGTH} characters",
-                    }
-                ],
-            }, 400
+        # Reject empty / over-length instructions at the boundary (shared with
+        # the streaming endpoint) before spending a planner+builder roundtrip.
+        guard = _workflow_instruction_guard(args)
+        if guard is not None:
+            return guard
 
         try:
             result = WorkflowGeneratorService.generate_workflow_graph(
@@ -380,3 +411,93 @@ class WorkflowGenerateApi(Resource):
             raise CompletionRequestError(e.description)
 
         return result
+
+
+@console_ns.route("/workflow-generate/suggestions")
+class WorkflowInstructionSuggestionsApi(Resource):
+    """Suggest short, buildable example instructions for the cmd+k generator.
+
+    Runs before a model is selected (uses the tenant's default model). The
+    underlying generator never raises, so an empty list is a valid 200 — the
+    frontend renders "no suggestions" rather than an error, so no provider-error
+    mapping is needed here.
+    """
+
+    @console_ns.doc("generate_workflow_instruction_suggestions")
+    @console_ns.doc(description="Suggest example workflow-generator instructions for the tenant")
+    @console_ns.expect(console_ns.models[WorkflowInstructionSuggestionsPayload.__name__])
+    @console_ns.response(200, "Suggestions generated successfully", console_ns.models[GeneratorResponse.__name__])
+    @console_ns.response(400, "Invalid request parameters")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @with_current_tenant_id
+    def post(self, current_tenant_id: str):
+        args = WorkflowInstructionSuggestionsPayload.model_validate(console_ns.payload)
+        suggestions = LLMGenerator.generate_workflow_instruction_suggestions(
+            tenant_id=current_tenant_id,
+            mode=args.mode,
+            language=args.language,
+            count=args.count,
+        )
+        return {"suggestions": suggestions}
+
+
+@console_ns.route("/workflow-generate/stream")
+class WorkflowGenerateStreamApi(Resource):
+    """Plan-first streaming variant of ``/workflow-generate`` (Server-Sent Events).
+
+    Emits a ``plan`` event (high-level node list + app metadata) as soon as the
+    planner returns, then a final ``result`` event with the full graph — the
+    SAME envelope ``/workflow-generate`` returns. Provider-init / invoke errors
+    are surfaced as a single ``result`` event (code ``MODEL_ERROR``) so the
+    frontend's stream parser always receives a result rather than a non-SSE HTTP
+    error.
+    """
+
+    @console_ns.doc("generate_workflow_graph_stream")
+    @console_ns.doc(description="Stream a Dify workflow graph (plan then result) via SSE")
+    @console_ns.expect(console_ns.models[WorkflowGeneratePayload.__name__])
+    @console_ns.response(200, "Server-Sent Events stream of plan/result events")
+    @console_ns.response(400, "Invalid request parameters")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @with_current_tenant_id
+    def post(self, current_tenant_id: str):
+        args = WorkflowGeneratePayload.model_validate(console_ns.payload)
+
+        # Same boundary guards as the blocking endpoint — return a normal 400
+        # JSON for these BEFORE opening the stream.
+        guard = _workflow_instruction_guard(args)
+        if guard is not None:
+            return guard
+
+        def generate() -> Generator[str, None, None]:
+            try:
+                for event_name, payload in WorkflowGeneratorService.generate_workflow_graph_stream(
+                    tenant_id=current_tenant_id,
+                    mode=args.mode,
+                    instruction=args.instruction,
+                    model_config=args.model_config_data,
+                    ideal_output=args.ideal_output,
+                    current_graph=args.current_graph,
+                ):
+                    body = {"event": event_name, **payload}
+                    yield f"data: {json.dumps(body)}\n\n"
+            except (ProviderTokenNotInitError, QuotaExceededError, ModelCurrentlyNotSupportError, InvokeError) as e:
+                # The model instance is resolved inside the service (lazily, on
+                # first iteration), so a provider / init error surfaces here.
+                # Emit it as a single SSE result event rather than a non-SSE
+                # error response so the frontend's stream parser always gets a
+                # result it can render.
+                detail = getattr(e, "description", None) or str(e) or "Model invocation failed"
+                error_body = {
+                    "event": "result",
+                    "graph": {"nodes": [], "edges": [], "viewport": {"x": 0.0, "y": 0.0, "zoom": 0.7}},
+                    "error": detail,
+                    "errors": [{"code": WorkflowGenerateErrorCode.MODEL_ERROR, "detail": detail}],
+                }
+                yield f"data: {json.dumps(error_body)}\n\n"
+
+        return compact_generate_response(generate())

api/controllers/console/app/mcp_server.py

@@ -26,6 +26,7 @@ from libs.helper import to_timestamp
 from libs.login import login_required
 from models.enums import AppMCPServerStatus
 from models.model import App, AppMCPServer
+from services.app_ref_service import AppRefService
 
 
 class MCPServerCreatePayload(BaseModel):
@@ -146,7 +147,17 @@ class AppMCPServerController(Resource):
     @get_app_model
     def put(self, app_model: App):
         payload = MCPServerUpdatePayload.model_validate(console_ns.payload or {})
-        server = db.session.get(AppMCPServer, payload.id)
+        app_ref = AppRefService.create_app_ref(app_model)
+        server_ref = AppRefService.create_mcp_server_ref(app_ref, payload.id)
+        server = db.session.scalar(
+            select(AppMCPServer)
+            .where(
+                AppMCPServer.id == server_ref.server_id,
+                AppMCPServer.tenant_id == server_ref.tenant_id,
+                AppMCPServer.app_id == server_ref.app_id,
+            )
+            .limit(1)
+        )
         if not server:
             raise NotFound()
 

api/controllers/console/app/message.py

@@ -13,7 +13,7 @@ from controllers.common.controller_schemas import MessageFeedbackPayload as _Mes
 from controllers.common.fields import SimpleResultResponse, TextFileResponse
 from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
 from controllers.console import console_ns
-from controllers.console.agent.app_helpers import resolve_agent_app_model
+from controllers.console.agent.app_helpers import resolve_agent_runtime_app_model
 from controllers.console.app.error import (
     CompletionRequestError,
     ProviderModelCurrentlyNotSupportError,
@@ -214,7 +214,7 @@ class AgentChatMessageListApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def get(self, current_tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
         return _list_chat_messages(app_model=app_model, current_user=current_user)
 
 
@@ -250,7 +250,7 @@ class AgentMessageFeedbackApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def post(self, current_tenant_id: str, current_user: Account, agent_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
         return _update_message_feedback(current_user=current_user, app_model=app_model)
 
 
@@ -315,7 +315,7 @@ class AgentMessageSuggestedQuestionApi(Resource):
     @with_current_user
     @with_current_tenant_id
     def get(self, current_tenant_id: str, current_user: Account, agent_id: UUID, message_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
         return _get_message_suggested_questions(current_user=current_user, app_model=app_model, message_id=message_id)
 
 
@@ -393,7 +393,7 @@ class AgentMessageApi(Resource):
     @account_initialization_required
     @with_current_tenant_id
     def get(self, current_tenant_id: str, agent_id: UUID, message_id: UUID):
-        app_model = resolve_agent_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
+        app_model = resolve_agent_runtime_app_model(tenant_id=current_tenant_id, agent_id=agent_id)
         return _get_message_detail(app_model=app_model, message_id=message_id)
 
 

api/controllers/console/app/site.py

@@ -40,6 +40,7 @@ class AppSiteUpdatePayload(BaseModel):
     customize_domain: str | None = Field(default=None)
     copyright: str | None = Field(default=None)
     privacy_policy: str | None = Field(default=None)
+    input_placeholder: str | None = Field(default=None)
     custom_disclaimer: str | None = Field(default=None)
     customize_token_strategy: Literal["must", "allow", "not_allow"] | None = Field(default=None)
     prompt_public: bool | None = Field(default=None)
@@ -66,6 +67,7 @@ class AppSiteResponse(ResponseModel):
     customize_domain: str | None = None
     copyright: str | None = None
     privacy_policy: str | None = None
+    input_placeholder: str | None = None
     custom_disclaimer: str | None = None
     customize_token_strategy: str
     prompt_public: bool
@@ -110,6 +112,7 @@ class AppSite(Resource):
             "customize_domain",
             "copyright",
             "privacy_policy",
+            "input_placeholder",
             "custom_disclaimer",
             "customize_token_strategy",
             "prompt_public",

api/controllers/console/app/workflow.py

@@ -6,7 +6,7 @@ from typing import Any, NotRequired, TypedDict, cast
 
 from flask import abort, request
 from flask_restx import Resource, fields
-from pydantic import AliasChoices, BaseModel, Field, RootModel, ValidationError, field_validator
+from pydantic import AliasChoices, BaseModel, ConfigDict, Field, RootModel, ValidationError, field_validator
 from sqlalchemy.orm import Session, sessionmaker
 from werkzeug.exceptions import BadRequest, Forbidden, InternalServerError, NotFound
 
@@ -78,6 +78,7 @@ from repositories.workflow_collaboration_repository import WORKFLOW_ONLINE_USERS
 from services.app_generate_service import AppGenerateService
 from services.errors.app import IsDraftWorkflowError, WorkflowHashNotEqualError, WorkflowNotFoundError
 from services.errors.llm import InvokeRateLimitError
+from services.workflow_ref_service import WorkflowRefService
 from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService
 
 logger = logging.getLogger(__name__)
@@ -158,8 +159,71 @@ class ConvertToWorkflowPayload(BaseModel):
     icon_background: str | None = None
 
 
+class WorkflowFeatureTogglePayload(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    enabled: bool | None = None
+
+
+class WorkflowSuggestedQuestionsAfterAnswerPayload(WorkflowFeatureTogglePayload):
+    model: dict[str, Any] | None = None
+    prompt: str | None = None
+
+
+class WorkflowTextToSpeechPayload(WorkflowFeatureTogglePayload):
+    language: str | None = None
+    voice: str | None = None
+    autoPlay: str | None = None
+
+
+class WorkflowSensitiveWordAvoidancePayload(WorkflowFeatureTogglePayload):
+    type: str | None = None
+    config: dict[str, Any] | None = None
+
+
+class WorkflowFileUploadTransferPayload(WorkflowFeatureTogglePayload):
+    number_limits: int | None = None
+    transfer_methods: list[str] | None = None
+
+
+class WorkflowFileUploadImagePayload(WorkflowFileUploadTransferPayload):
+    detail: str | None = None
+
+
+class WorkflowFileUploadPreviewConfigPayload(BaseModel):
+    mode: str | None = None
+    file_type_list: list[str] | None = None
+
+
+class WorkflowFileUploadPayload(WorkflowFeatureTogglePayload):
+    allowed_file_types: list[str] | None = None
+    allowed_file_extensions: list[str] | None = None
+    allowed_file_upload_methods: list[str] | None = None
+    number_limits: int | None = None
+    image: WorkflowFileUploadImagePayload | None = None
+    document: WorkflowFileUploadTransferPayload | None = None
+    audio: WorkflowFileUploadTransferPayload | None = None
+    video: WorkflowFileUploadTransferPayload | None = None
+    custom: WorkflowFileUploadTransferPayload | None = None
+    preview_config: WorkflowFileUploadPreviewConfigPayload | None = None
+    fileUploadConfig: dict[str, Any] | None = None
+
+
+class WorkflowFeaturesConfigPayload(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    opening_statement: str | None = None
+    suggested_questions: list[str] | None = None
+    suggested_questions_after_answer: WorkflowSuggestedQuestionsAfterAnswerPayload | None = None
+    text_to_speech: WorkflowTextToSpeechPayload | None = None
+    speech_to_text: WorkflowFeatureTogglePayload | None = None
+    retriever_resource: WorkflowFeatureTogglePayload | None = None
+    sensitive_word_avoidance: WorkflowSensitiveWordAvoidancePayload | None = None
+    file_upload: WorkflowFileUploadPayload | None = None
+
+
 class WorkflowFeaturesPayload(BaseModel):
-    features: dict[str, Any] = Field(
+    features: WorkflowFeaturesConfigPayload = Field(
         ...,
         description="Workflow feature configuration",
     )
@@ -343,6 +407,15 @@ register_schema_models(
     ConvertToWorkflowPayload,
     WorkflowListQuery,
     WorkflowUpdatePayload,
+    WorkflowFeatureTogglePayload,
+    WorkflowSuggestedQuestionsAfterAnswerPayload,
+    WorkflowTextToSpeechPayload,
+    WorkflowSensitiveWordAvoidancePayload,
+    WorkflowFileUploadTransferPayload,
+    WorkflowFileUploadImagePayload,
+    WorkflowFileUploadPreviewConfigPayload,
+    WorkflowFileUploadPayload,
+    WorkflowFeaturesConfigPayload,
     WorkflowFeaturesPayload,
     WorkflowOnlineUsersPayload,
     DraftWorkflowTriggerRunPayload,
@@ -1274,7 +1347,7 @@ class WorkflowFeaturesApi(Resource):
     def post(self, current_user: Account, app_model: App):
 
         args = WorkflowFeaturesPayload.model_validate(console_ns.payload or {})
-        features = args.features
+        features = args.features.model_dump(mode="json", exclude_unset=True)
 
         workflow_service = WorkflowService()
         workflow_service.update_draft_workflow_features(app_model=app_model, features=features, account=current_user)
@@ -1406,15 +1479,15 @@ class WorkflowByIdApi(Resource):
             return {"message": "No valid fields to update"}, 400
 
         workflow_service = WorkflowService()
+        workflow_ref = WorkflowRefService.create_app_workflow_ref(app_model, workflow_id)
 
         # Create a session and manage the transaction
         with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
             workflow = workflow_service.update_workflow(
                 session=session,
-                workflow_id=workflow_id,
-                tenant_id=app_model.tenant_id,
                 account_id=current_user.id,
                 data=update_data,
+                workflow_ref=workflow_ref,
             )
 
             if not workflow:
@@ -1434,12 +1507,14 @@ class WorkflowByIdApi(Resource):
         Delete workflow
         """
         workflow_service = WorkflowService()
+        workflow_ref = WorkflowRefService.create_app_workflow_ref(app_model, workflow_id)
 
         # Create a session and manage the transaction
         with sessionmaker(db.engine).begin() as session:
             try:
                 workflow_service.delete_workflow(
-                    session=session, workflow_id=workflow_id, tenant_id=app_model.tenant_id
+                    session=session,
+                    workflow_ref=workflow_ref,
                 )
             except WorkflowInUseError as e:
                 abort(400, description=str(e))

api/controllers/console/app/workflow_comment.py

@@ -189,14 +189,14 @@ class WorkflowCommentReplyUpdate(ResponseModel):
 
 register_schema_models(
     console_ns,
-    AccountWithRole,
-    WorkflowCommentMentionUsersPayload,
     WorkflowCommentCreatePayload,
     WorkflowCommentUpdatePayload,
     WorkflowCommentReplyPayload,
 )
 register_response_schema_models(
     console_ns,
+    AccountWithRole,
+    WorkflowCommentMentionUsersPayload,
     WorkflowCommentAccount,
     WorkflowCommentReply,
     WorkflowCommentMention,

api/controllers/console/app/workflow_draft_variable.py

@@ -6,7 +6,7 @@ from uuid import UUID
 
 from flask import Response, request
 from flask_restx import Resource, fields, marshal, marshal_with
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 from sqlalchemy.orm import sessionmaker
 
 from controllers.common.errors import InvalidArgumentError, NotFoundError
@@ -79,15 +79,33 @@ class WorkflowDraftVariableUpdatePayload(BaseModel):
     value: Any | None = Field(default=None, description="Variable value")
 
 
+class WorkflowVariableItemPayload(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    id: str | None = None
+    name: str | None = None
+    value_type: str | None = None
+    value: Any | None = None
+    description: str | None = None
+
+
+class ConversationVariableItemPayload(WorkflowVariableItemPayload):
+    pass
+
+
+class EnvironmentVariableItemPayload(WorkflowVariableItemPayload):
+    pass
+
+
 class ConversationVariableUpdatePayload(BaseModel):
-    conversation_variables: list[dict[str, Any]] = Field(
+    conversation_variables: list[ConversationVariableItemPayload] = Field(
         ...,
         description="Conversation variables for the draft workflow",
     )
 
 
 class EnvironmentVariableUpdatePayload(BaseModel):
-    environment_variables: list[dict[str, Any]] = Field(
+    environment_variables: list[EnvironmentVariableItemPayload] = Field(
         ...,
         description="Environment variables for the draft workflow",
     )
@@ -114,7 +132,9 @@ register_schema_models(
     console_ns,
     WorkflowDraftVariableListQuery,
     WorkflowDraftVariableUpdatePayload,
+    ConversationVariableItemPayload,
     ConversationVariableUpdatePayload,
+    EnvironmentVariableItemPayload,
     EnvironmentVariableUpdatePayload,
 )
 register_response_schema_models(console_ns, SimpleResultResponse, EnvironmentVariableListResponse)
@@ -615,7 +635,9 @@ class ConversationVariableCollectionApi(Resource):
 
         workflow_service = WorkflowService()
 
-        conversation_variables_list = payload.conversation_variables
+        conversation_variables_list = [
+            variable.model_dump(mode="json", exclude_unset=True) for variable in payload.conversation_variables
+        ]
         conversation_variables = [
             variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list
         ]
@@ -707,7 +729,9 @@ class EnvironmentVariableCollectionApi(Resource):
 
         workflow_service = WorkflowService()
 
-        environment_variables_list = payload.environment_variables
+        environment_variables_list = [
+            variable.model_dump(mode="json", exclude_unset=True) for variable in payload.environment_variables
+        ]
         environment_variables = [
             variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list
         ]

api/controllers/console/billing/billing.py

@@ -16,6 +16,8 @@ from controllers.console.wraps import (
     with_current_user,
 )
 from enums.cloud_plan import CloudPlan
+from extensions.ext_database import db
+from fields.base import ResponseModel
 from libs.login import login_required
 from models import Account
 from services.billing_service import BillingService
@@ -34,8 +36,12 @@ class BillingResponse(RootModel[dict[str, Any]]):
     root: dict[str, Any]
 
 
+class BillingInvoiceResponse(ResponseModel):
+    url: str
+
+
 register_schema_models(console_ns, SubscriptionQuery, PartnerTenantsPayload)
-register_response_schema_models(console_ns, BillingResponse)
+register_response_schema_models(console_ns, BillingResponse, BillingInvoiceResponse)
 
 
 @console_ns.route("/billing/subscription")
@@ -50,13 +56,13 @@ class Subscription(Resource):
     @with_current_tenant_id
     def get(self, current_tenant_id: str, current_user: Account):
         args = SubscriptionQuery.model_validate(request.args.to_dict(flat=True))
-        BillingService.is_tenant_owner_or_admin(current_user)
+        BillingService.is_tenant_owner_or_admin(db.session, current_user)
         return BillingService.get_subscription(args.plan, args.interval, current_user.email, current_tenant_id)
 
 
 @console_ns.route("/billing/invoices")
 class Invoices(Resource):
-    @console_ns.response(200, "Success", console_ns.models[BillingResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[BillingInvoiceResponse.__name__])
     @setup_required
     @login_required
     @account_initialization_required
@@ -64,7 +70,7 @@ class Invoices(Resource):
     @with_current_user
     @with_current_tenant_id
     def get(self, current_tenant_id: str, current_user: Account):
-        BillingService.is_tenant_owner_or_admin(current_user)
+        BillingService.is_tenant_owner_or_admin(db.session, current_user)
         return BillingService.get_invoices(current_user.email, current_tenant_id)
 
 

api/controllers/console/datasets/data_source.py

@@ -243,72 +243,71 @@ class DataSourceNotionListApi(Resource):
         if not credential:
             raise NotFound("Credential not found.")
         exist_page_ids = []
-        with sessionmaker(db.engine).begin() as session:
-            # import notion in the exist dataset
-            if query.dataset_id:
-                dataset = DatasetService.get_dataset(query.dataset_id)
-                if not dataset:
-                    raise NotFound("Dataset not found.")
-                if dataset.data_source_type != "notion_import":
-                    raise ValueError("Dataset is not notion type.")
+        # import notion in the exist dataset
+        if query.dataset_id:
+            dataset = DatasetService.get_dataset(query.dataset_id, db.session)
+            if not dataset:
+                raise NotFound("Dataset not found.")
+            if dataset.data_source_type != "notion_import":
+                raise ValueError("Dataset is not notion type.")
 
-                documents = session.scalars(
-                    select(Document).where(
-                        Document.dataset_id == query.dataset_id,
-                        Document.tenant_id == current_tenant_id,
-                        Document.data_source_type == "notion_import",
-                        Document.enabled.is_(True),
-                    )
-                ).all()
-                if documents:
-                    for document in documents:
-                        data_source_info = json.loads(document.data_source_info)
-                        exist_page_ids.append(data_source_info["notion_page_id"])
-            # get all authorized pages
-            from core.datasource.datasource_manager import DatasourceManager
-
-            datasource_runtime = DatasourceManager.get_datasource_runtime(
-                provider_id="langgenius/notion_datasource/notion_datasource",
-                datasource_name="notion_datasource",
-                tenant_id=current_tenant_id,
-                datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,
-            )
-            datasource_provider_service = DatasourceProviderService()
-            if credential:
-                datasource_runtime.runtime.credentials = credential
-            datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime)
-            online_document_result: Generator[OnlineDocumentPagesMessage, None, None] = (
-                datasource_runtime.get_online_document_pages(
-                    user_id=current_user.id,
-                    datasource_parameters={},
-                    provider_type=datasource_runtime.datasource_provider_type(),
+            documents = db.session.scalars(
+                select(Document).where(
+                    Document.dataset_id == query.dataset_id,
+                    Document.tenant_id == current_tenant_id,
+                    Document.data_source_type == "notion_import",
+                    Document.enabled.is_(True),
                 )
+            ).all()
+            if documents:
+                for document in documents:
+                    data_source_info = json.loads(document.data_source_info)
+                    exist_page_ids.append(data_source_info["notion_page_id"])
+        # get all authorized pages
+        from core.datasource.datasource_manager import DatasourceManager
+
+        datasource_runtime = DatasourceManager.get_datasource_runtime(
+            provider_id="langgenius/notion_datasource/notion_datasource",
+            datasource_name="notion_datasource",
+            tenant_id=current_tenant_id,
+            datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,
+        )
+        datasource_provider_service = DatasourceProviderService()
+        if credential:
+            datasource_runtime.runtime.credentials = credential
+        datasource_runtime = cast(OnlineDocumentDatasourcePlugin, datasource_runtime)
+        online_document_result: Generator[OnlineDocumentPagesMessage, None, None] = (
+            datasource_runtime.get_online_document_pages(
+                user_id=current_user.id,
+                datasource_parameters={},
+                provider_type=datasource_runtime.datasource_provider_type(),
             )
-            try:
-                pages = []
-                workspace_info = {}
-                for message in online_document_result:
-                    result = message.result
-                    for info in result:
-                        workspace_info = {
-                            "workspace_id": info.workspace_id,
-                            "workspace_name": info.workspace_name,
-                            "workspace_icon": info.workspace_icon,
+        )
+        try:
+            pages = []
+            workspace_info = {}
+            for message in online_document_result:
+                result = message.result
+                for info in result:
+                    workspace_info = {
+                        "workspace_id": info.workspace_id,
+                        "workspace_name": info.workspace_name,
+                        "workspace_icon": info.workspace_icon,
+                    }
+                    for page in info.pages:
+                        page_info = {
+                            "page_id": page.page_id,
+                            "page_name": page.page_name,
+                            "type": page.type,
+                            "parent_id": page.parent_id,
+                            "is_bound": page.page_id in exist_page_ids,
+                            "page_icon": page.page_icon,
                         }
-                        for page in info.pages:
-                            page_info = {
-                                "page_id": page.page_id,
-                                "page_name": page.page_name,
-                                "type": page.type,
-                                "parent_id": page.parent_id,
-                                "is_bound": page.page_id in exist_page_ids,
-                                "page_icon": page.page_icon,
-                            }
-                            pages.append(page_info)
-            except Exception as e:
-                raise e
-            notion_info = [{**workspace_info, "pages": pages}] if workspace_info else []
-            return dump_response(NotionIntegrateInfoListResponse, {"notion_info": notion_info}), 200
+                        pages.append(page_info)
+        except Exception as e:
+            raise e
+        notion_info = [{**workspace_info, "pages": pages}] if workspace_info else []
+        return dump_response(NotionIntegrateInfoListResponse, {"notion_info": notion_info}), 200
 
 
 @console_ns.route("/notion/pages/<uuid:page_id>/<string:page_type>/preview")
@@ -401,11 +400,11 @@ class DataSourceNotionDatasetSyncApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_CREATE_AND_MANAGEMENT)
     def get(self, dataset_id: UUID) -> tuple[dict[str, str], int]:
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
-        documents = DocumentService.get_document_by_dataset_id(dataset_id_str)
+        documents = DocumentService.get_document_by_dataset_id(dataset_id_str, db.session)
         for document in documents:
             document_indexing_sync_task.delay(dataset_id_str, document.id)
         return {"result": "success"}, 200
@@ -421,11 +420,11 @@ class DataSourceNotionDocumentSyncApi(Resource):
     def get(self, dataset_id: UUID, document_id: UUID) -> tuple[dict[str, str], int]:
         dataset_id_str = str(dataset_id)
         document_id_str = str(document_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if document is None:
             raise NotFound("Document not found.")
         document_indexing_sync_task.delay(dataset_id_str, document_id_str)

api/controllers/console/datasets/datasets.py

@@ -561,6 +561,7 @@ class DatasetListApi(Resource):
                 provider=payload.provider,
                 external_knowledge_api_id=payload.external_knowledge_api_id,
                 external_knowledge_id=payload.external_knowledge_id,
+                session=db.session,
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()
@@ -598,11 +599,11 @@ class DatasetApi(Resource):
     @with_current_tenant_id
     def get(self, current_tenant_id: str, current_user: Account, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
         permissions = enterprise_rbac_service.RBACService.MyPermissions.get(
@@ -618,7 +619,7 @@ class DatasetApi(Resource):
                 provider_id = ModelProviderID(dataset.embedding_model_provider)
                 data["embedding_model_provider"] = str(provider_id)
         if data.get("permission") == "partial_members":
-            part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+            part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
             data.update({"partial_member_list": part_users_list})
 
         # check embedding setting
@@ -661,7 +662,7 @@ class DatasetApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_EDIT)
     def patch(self, current_tenant_id: str, current_user: Account, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
@@ -680,10 +681,10 @@ class DatasetApi(Resource):
         # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
         if not dify_config.RBAC_ENABLED:
             DatasetPermissionService.check_permission(
-                current_user, dataset, payload.permission, payload.partial_member_list
+                current_user, dataset, payload.permission, payload.partial_member_list, db.session
             )
 
-        dataset = DatasetService.update_dataset(dataset_id_str, payload_data, current_user)
+        dataset = DatasetService.update_dataset(dataset_id_str, payload_data, current_user, db.session)
 
         if dataset is None:
             raise NotFound("Dataset not found.")
@@ -698,12 +699,14 @@ class DatasetApi(Resource):
         tenant_id = current_tenant_id
 
         if payload.partial_member_list is not None and payload.permission == DatasetPermissionEnum.PARTIAL_TEAM:
-            DatasetPermissionService.update_partial_member_list(tenant_id, dataset_id_str, payload.partial_member_list)
+            DatasetPermissionService.update_partial_member_list(
+                tenant_id, dataset_id_str, payload.partial_member_list, db.session
+            )
         # clear partial member list when permission is only_me or all_team_members
         elif payload.permission in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:
-            DatasetPermissionService.clear_partial_member_list(dataset_id_str)
+            DatasetPermissionService.clear_partial_member_list(dataset_id_str, db.session)
 
-        partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+        partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
         result_data.update({"partial_member_list": partial_member_list})
 
         return result_data, 200
@@ -722,8 +725,8 @@ class DatasetApi(Resource):
             raise Forbidden()
 
         try:
-            if DatasetService.delete_dataset(dataset_id_str, current_user):
-                DatasetPermissionService.clear_partial_member_list(dataset_id_str)
+            if DatasetService.delete_dataset(dataset_id_str, current_user, db.session):
+                DatasetPermissionService.clear_partial_member_list(dataset_id_str, db.session)
                 return "", 204
             else:
                 raise NotFound("Dataset not found.")
@@ -748,7 +751,7 @@ class DatasetUseCheckApi(Resource):
     def get(self, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
 
-        dataset_is_using = DatasetService.dataset_use_check(dataset_id_str)
+        dataset_is_using = DatasetService.dataset_use_check(dataset_id_str, db.session)
         return {"is_using": dataset_is_using}, 200
 
 
@@ -769,12 +772,12 @@ class DatasetQueryApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
     def get(self, current_user: Account, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
@@ -910,16 +913,16 @@ class DatasetRelatedAppListApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
     def get(self, current_user: Account, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
-        app_dataset_joins = DatasetService.get_related_apps(dataset.id)
+        app_dataset_joins = DatasetService.get_related_apps(dataset.id, db.session)
 
         related_apps = []
         for app_dataset_join in app_dataset_joins:
@@ -1094,7 +1097,7 @@ class DatasetEnableApiApi(Resource):
     def post(self, dataset_id: UUID, status: str):
         dataset_id_str = str(dataset_id)
 
-        DatasetService.update_dataset_api_status(dataset_id_str, status == "enable")
+        DatasetService.update_dataset_api_status(dataset_id_str, status == "enable", db.session)
 
         return {"result": "success"}, 200
 
@@ -1163,10 +1166,10 @@ class DatasetErrorDocs(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
     def get(self, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        results = DocumentService.get_error_documents_by_dataset_id(dataset_id_str)
+        results = DocumentService.get_error_documents_by_dataset_id(dataset_id_str, db.session)
 
         return dump_response(ErrorDocsResponse, {"data": results, "total": len(results)}), 200
 
@@ -1190,15 +1193,15 @@ class DatasetPermissionUserListApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
     def get(self, current_user: Account, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
-        partial_members_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+        partial_members_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
 
         return dump_response(PartialMemberListResponse, {"data": partial_members_list}), 200
 
@@ -1220,7 +1223,8 @@ class DatasetAutoDisableLogApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_READONLY)
     def get(self, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        return dump_response(AutoDisableLogsResponse, DatasetService.get_dataset_auto_disable_logs(dataset_id_str)), 200
+        auto_disable_logs = DatasetService.get_dataset_auto_disable_logs(dataset_id_str, db.session)
+        return dump_response(AutoDisableLogsResponse, auto_disable_logs), 200

api/controllers/console/datasets/datasets_document.py

@@ -49,6 +49,7 @@ from libs.login import login_required
 from models import Account, DatasetProcessRule, Document, DocumentSegment, UploadFile
 from models.dataset import DocumentPipelineExecutionLog
 from models.enums import IndexingStatus, SegmentStatus
+from services.dataset_ref_service import DatasetRefService
 from services.dataset_service import DatasetService, DocumentService
 from services.entities.knowledge_entities.knowledge_entities import KnowledgeConfig, ProcessRule, RetrievalModel
 from services.file_service import FileService
@@ -181,16 +182,16 @@ class DocumentResource(Resource):
     def get_document(
         self, dataset_id: str, document_id: str, current_user: Account, current_tenant_id: str
     ) -> Document:
-        dataset = DatasetService.get_dataset(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
-        document = DocumentService.get_document(dataset_id, document_id)
+        document = DocumentService.get_document(dataset_id, document_id, session=db.session)
 
         if not document:
             raise NotFound("Document not found.")
@@ -201,16 +202,16 @@ class DocumentResource(Resource):
         return document
 
     def get_batch_documents(self, dataset_id: str, batch: str, current_user: Account) -> Sequence[Document]:
-        dataset = DatasetService.get_dataset(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
-        documents = DocumentService.get_batch_documents(dataset_id, batch)
+        documents = DocumentService.get_batch_documents(dataset_id, batch, db.session)
 
         if not documents:
             raise NotFound("Documents not found.")
@@ -241,13 +242,13 @@ class GetProcessRuleApi(Resource):
             # get the latest process rule
             document = db.get_or_404(Document, document_id)
 
-            dataset = DatasetService.get_dataset(document.dataset_id)
+            dataset = DatasetService.get_dataset(document.dataset_id, db.session)
 
             if not dataset:
                 raise NotFound("Dataset not found.")
 
             try:
-                DatasetService.check_dataset_permission(dataset, current_user)
+                DatasetService.check_dataset_permission(dataset, current_user, db.session)
             except services.errors.account.NoPermissionError as e:
                 raise Forbidden(str(e))
 
@@ -317,12 +318,12 @@ class DatasetDocumentListApi(Resource):
                     )
         except (ArgumentTypeError, ValueError, Exception):
             fetch = False
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
@@ -421,7 +422,7 @@ class DatasetDocumentListApi(Resource):
     def post(self, current_user: Account, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
 
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
 
         if not dataset:
             raise NotFound("Dataset not found.")
@@ -431,7 +432,7 @@ class DatasetDocumentListApi(Resource):
             raise Forbidden()
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
@@ -444,8 +445,10 @@ class DatasetDocumentListApi(Resource):
         DocumentService.document_create_args_validate(knowledge_config)
 
         try:
-            documents, batch = DocumentService.save_document_with_dataset_id(dataset, knowledge_config, current_user)
-            dataset = DatasetService.get_dataset(dataset_id_str)
+            documents, batch = DocumentService.save_document_with_dataset_id(
+                dataset, knowledge_config, current_user, session=db.session
+            )
+            dataset = DatasetService.get_dataset(dataset_id_str, db.session)
 
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -464,7 +467,7 @@ class DatasetDocumentListApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_EDIT)
     def delete(self, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
         # check user's model setting
@@ -472,7 +475,8 @@ class DatasetDocumentListApi(Resource):
 
         try:
             document_ids = request.args.getlist("document_id")
-            DocumentService.delete_documents(dataset, document_ids)
+            dataset_ref = DatasetRefService.create_dataset_ref(dataset)
+            DocumentService.delete_documents(dataset_ref, document_ids, dataset.doc_form, db.session)
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError("Cannot delete document during indexing.")
 
@@ -531,6 +535,7 @@ class DatasetInitApi(Resource):
                 tenant_id=current_tenant_id,
                 knowledge_config=knowledge_config,
                 account=current_user,
+                session=db.session,
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -867,7 +872,7 @@ class DocumentApi(DocumentResource):
         if metadata == "only":
             response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
         elif metadata == "without":
-            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str)
+            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session)
             document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
             response = {
                 "id": document.id,
@@ -901,7 +906,7 @@ class DocumentApi(DocumentResource):
                 "need_summary": document.need_summary if document.need_summary is not None else False,
             }
         else:
-            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str)
+            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session)
             document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
             response = {
                 "id": document.id,
@@ -950,7 +955,7 @@ class DocumentApi(DocumentResource):
     def delete(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):
         dataset_id_str = str(dataset_id)
         document_id_str = str(document_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
         # check user's model setting
@@ -959,7 +964,7 @@ class DocumentApi(DocumentResource):
         document = self.get_document(dataset_id_str, document_id_str, current_user, current_tenant_id)
 
         try:
-            DocumentService.delete_document(document)
+            DocumentService.delete_document(document, db.session)
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError("Cannot delete document during indexing.")
 
@@ -983,7 +988,7 @@ class DocumentDownloadApi(DocumentResource):
     def get(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID) -> dict[str, Any]:
         # Reuse the shared permission/tenant checks implemented in DocumentResource.
         document = self.get_document(str(dataset_id), str(document_id), current_user, current_tenant_id)
-        return {"url": DocumentService.get_document_download_url(document)}
+        return {"url": DocumentService.get_document_download_url(document, db.session)}
 
 
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/download-zip")
@@ -1013,6 +1018,7 @@ class DocumentBatchDownloadZipApi(DocumentResource):
             document_ids=document_ids,
             tenant_id=current_tenant_id,
             current_user=current_user,
+            session=db.session,
         )
 
         # Delegate ZIP packing to FileService, but keep Flask response+cleanup in the route.
@@ -1161,7 +1167,7 @@ class DocumentStatusApi(DocumentResource):
         self, current_user: Account, dataset_id: UUID, action: Literal["enable", "disable", "archive", "un_archive"]
     ):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
@@ -1173,12 +1179,12 @@ class DocumentStatusApi(DocumentResource):
         DatasetService.check_dataset_model_setting(dataset)
 
         # check user's permission
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         document_ids = request.args.getlist("document_id")
 
         try:
-            DocumentService.batch_update_document_status(dataset, document_ids, action, current_user)
+            DocumentService.batch_update_document_status(dataset, document_ids, action, current_user, db.session)
         except services.errors.document.DocumentIndexingError as e:
             raise InvalidActionError(str(e))
         except ValueError as e:
@@ -1202,11 +1208,11 @@ class DocumentPauseApi(DocumentResource):
         dataset_id_str = str(dataset_id)
         document_id_str = str(document_id)
 
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
 
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
 
         # 404 if document not found
         if document is None:
@@ -1218,7 +1224,7 @@ class DocumentPauseApi(DocumentResource):
 
         try:
             # pause document
-            DocumentService.pause_document(document)
+            DocumentService.pause_document(document, db.session)
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError("Cannot pause completed document.")
 
@@ -1237,10 +1243,10 @@ class DocumentRecoverApi(DocumentResource):
         """recover document."""
         dataset_id_str = str(dataset_id)
         document_id_str = str(document_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
 
         # 404 if document not found
         if document is None:
@@ -1251,7 +1257,7 @@ class DocumentRecoverApi(DocumentResource):
             raise ArchivedDocumentImmutableError()
         try:
             # pause document
-            DocumentService.recover_document(document)
+            DocumentService.recover_document(document, db.session)
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError("Document is not in paused status.")
 
@@ -1271,13 +1277,13 @@ class DocumentRetryApi(DocumentResource):
         """retry document."""
         payload = DocumentRetryPayload.model_validate(console_ns.payload or {})
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         retry_documents = []
         if not dataset:
             raise NotFound("Dataset not found.")
         for document_id in payload.document_ids:
             try:
-                document = DocumentService.get_document(dataset.id, document_id)
+                document = DocumentService.get_document(dataset.id, document_id, session=db.session)
 
                 # 404 if document not found
                 if document is None:
@@ -1295,7 +1301,7 @@ class DocumentRetryApi(DocumentResource):
                 logger.exception("Failed to retry document, document id: %s", document_id)
                 continue
         # retry document
-        DocumentService.retry_document(dataset_id_str, retry_documents)
+        DocumentService.retry_document(dataset_id_str, retry_documents, db.session)
 
         return "", 204
 
@@ -1313,14 +1319,14 @@ class DocumentRenameApi(DocumentResource):
         # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
         if not current_user.is_dataset_editor:
             raise Forbidden()
-        dataset = DatasetService.get_dataset(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_operator_permission(current_user, dataset)
+        DatasetService.check_dataset_operator_permission(current_user, dataset, session=db.session)
         payload = DocumentRenamePayload.model_validate(console_ns.payload or {})
 
         try:
-            document = DocumentService.rename_document(str(dataset_id), str(document_id), payload.name)
+            document = DocumentService.rename_document(str(dataset_id), str(document_id), payload.name, db.session)
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError("Cannot delete document during indexing.")
 
@@ -1338,11 +1344,11 @@ class WebsiteDocumentSyncApi(DocumentResource):
     def get(self, current_tenant_id: str, dataset_id: UUID, document_id: UUID):
         """sync website document."""
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         if document.tenant_id != current_tenant_id:
@@ -1353,7 +1359,7 @@ class WebsiteDocumentSyncApi(DocumentResource):
         if DocumentService.check_archived(document):
             raise ArchivedDocumentImmutableError()
         # sync document
-        DocumentService.sync_website_document(dataset_id_str, document)
+        DocumentService.sync_website_document(dataset_id_str, document, db.session)
 
         return {"result": "success"}, 200
 
@@ -1373,10 +1379,10 @@ class DocumentPipelineExecutionLogApi(DocumentResource):
         dataset_id_str = str(dataset_id)
         document_id_str = str(document_id)
 
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         log = db.session.scalar(
@@ -1431,7 +1437,7 @@ class DocumentGenerateSummaryApi(Resource):
         dataset_id_str = str(dataset_id)
 
         # Get dataset
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
 
@@ -1440,7 +1446,7 @@ class DocumentGenerateSummaryApi(Resource):
             raise Forbidden()
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
@@ -1465,7 +1471,7 @@ class DocumentGenerateSummaryApi(Resource):
             raise ValueError("Summary index is not enabled for this dataset. Please enable it in the dataset settings.")
 
         # Verify all documents exist and belong to the dataset
-        documents = DocumentService.get_documents_by_ids(dataset_id_str, document_list)
+        documents = DocumentService.get_documents_by_ids(dataset_id_str, document_list, db.session)
 
         if len(documents) != len(document_list):
             found_ids = {doc.id for doc in documents}
@@ -1481,6 +1487,7 @@ class DocumentGenerateSummaryApi(Resource):
             DocumentService.update_documents_need_summary(
                 dataset_id=dataset_id_str,
                 document_ids=document_ids_to_update,
+                session=db.session,
                 need_summary=True,
             )
 
@@ -1531,13 +1538,13 @@ class DocumentSummaryStatusApi(DocumentResource):
         document_id_str = str(document_id)
 
         # Get dataset
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
 
         # Check permissions
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
@@ -1547,6 +1554,7 @@ class DocumentSummaryStatusApi(DocumentResource):
         result = SummaryIndexService.get_document_summary_status_detail(
             document_id=document_id_str,
             dataset_id=dataset_id_str,
+            session=db.session,
         )
 
         return result, 200

api/controllers/console/datasets/datasets_segments.py

@@ -58,8 +58,9 @@ from graphon.model_runtime.entities.model_entities import ModelType
 from libs.helper import dump_response, escape_like_pattern
 from libs.login import login_required
 from models import Account
-from models.dataset import ChildChunk, DocumentSegment
+from models.dataset import Dataset, Document, DocumentSegment
 from models.model import UploadFile
+from services.dataset_ref_service import DatasetRefService, SegmentRef
 from services.dataset_service import DatasetService, DocumentService, SegmentService
 from services.entities.knowledge_entities.knowledge_entities import ChildChunkUpdateArgs, SegmentUpdateArgs
 from services.errors.chunk import ChildChunkDeleteIndexError as ChildChunkDeleteIndexServiceError
@@ -162,6 +163,21 @@ register_response_schema_models(
 )
 
 
+def _get_segment_for_document(
+    dataset: Dataset, document: Document, segment_id: str
+) -> tuple[SegmentRef, DocumentSegment]:
+    dataset_ref = DatasetRefService.create_dataset_ref(dataset)
+    document_ref = DatasetRefService.create_document_ref(dataset_ref, document)
+    if document_ref is None:
+        raise NotFound("Document not found.")
+
+    segment_ref = DatasetRefService.create_segment_ref(document_ref, segment_id)
+    segment = SegmentService.get_segment_by_ref(segment_ref)
+    if not segment:
+        raise NotFound("Segment not found.")
+    return segment_ref, segment
+
+
 @console_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
 class DatasetDocumentSegmentListApi(Resource):
     @console_ns.doc(params=SegmentDocParams.DATASET_DOCUMENT)
@@ -176,16 +192,16 @@ class DatasetDocumentSegmentListApi(Resource):
     def get(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):
         dataset_id_str = str(dataset_id)
         document_id_str = str(document_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
 
         if not document:
             raise NotFound("Document not found.")
@@ -286,14 +302,14 @@ class DatasetDocumentSegmentListApi(Resource):
     def delete(self, current_user: Account, dataset_id: UUID, document_id: UUID):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         segment_ids = request.args.getlist("segment_id")
@@ -302,10 +318,10 @@ class DatasetDocumentSegmentListApi(Resource):
         if not current_user.is_dataset_editor:
             raise Forbidden()
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
-        SegmentService.delete_segments(segment_ids, document, dataset)
+        SegmentService.delete_segments(segment_ids, document, dataset, db.session)
         return "", 204
 
 
@@ -331,11 +347,11 @@ class DatasetDocumentSegmentApi(Resource):
         action: Literal["enable", "disable"],
     ):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         # check user's model setting
@@ -345,7 +361,7 @@ class DatasetDocumentSegmentApi(Resource):
             raise Forbidden()
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
         if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
@@ -371,7 +387,7 @@ class DatasetDocumentSegmentApi(Resource):
         if cache_result is not None:
             raise InvalidActionError("Document is being indexed, please try again later")
         try:
-            SegmentService.update_segments_status(segment_ids, action, dataset, document)
+            SegmentService.update_segments_status(segment_ids, action, dataset, document, db.session)
         except Exception as e:
             raise InvalidActionError(str(e))
         return dump_response(SimpleResultResponse, {"result": "success"}), 200
@@ -394,12 +410,12 @@ class DatasetDocumentSegmentAddApi(Resource):
     def post(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         if not current_user.is_dataset_editor:
@@ -421,14 +437,14 @@ class DatasetDocumentSegmentAddApi(Resource):
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
         # validate args
         payload = SegmentCreatePayload.model_validate(console_ns.payload or {})
         payload_dict = payload.model_dump(exclude_none=True)
         SegmentService.segment_create_args_validate(payload_dict, document)
-        segment = type_cast(DocumentSegment, SegmentService.create_segment(payload_dict, document, dataset))
+        segment = type_cast(DocumentSegment, SegmentService.create_segment(payload_dict, document, dataset, db.session))
         summary = SummaryIndexService.get_segment_summary(segment_id=segment.id, dataset_id=dataset_id_str)
         response = {
             "data": segment_response_with_summary(segment, summary.summary_content if summary else None),
@@ -455,16 +471,23 @@ class DatasetDocumentSegmentUpdateApi(Resource):
     ):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
+        # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
+        if not current_user.is_dataset_editor:
+            raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
         if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
             # check embedding model setting
             try:
@@ -481,22 +504,8 @@ class DatasetDocumentSegmentUpdateApi(Resource):
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
-            # check segment
         segment_id_str = str(segment_id)
-        segment = db.session.scalar(
-            select(DocumentSegment)
-            .where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
-            .limit(1)
-        )
-        if not segment:
-            raise NotFound("Segment not found.")
-        # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
-        if not current_user.is_dataset_editor:
-            raise Forbidden()
-        try:
-            DatasetService.check_dataset_permission(dataset, current_user)
-        except services.errors.account.NoPermissionError as e:
-            raise Forbidden(str(e))
+        _, segment = _get_segment_for_document(dataset, document, segment_id_str)
         # validate args
         payload = SegmentUpdatePayload.model_validate(console_ns.payload or {})
         payload_dict = payload.model_dump(exclude_none=True)
@@ -504,7 +513,11 @@ class DatasetDocumentSegmentUpdateApi(Resource):
 
         # Update segment (summary update with change detection is handled in SegmentService.update_segment)
         segment = SegmentService.update_segment(
-            SegmentUpdateArgs.model_validate(payload.model_dump(exclude_none=True)), segment, document, dataset
+            SegmentUpdateArgs.model_validate(payload.model_dump(exclude_none=True)),
+            segment,
+            document,
+            dataset,
+            db.session,
         )
         summary = SummaryIndexService.get_segment_summary(segment_id=segment.id, dataset_id=dataset_id_str)
         response = {
@@ -527,33 +540,26 @@ class DatasetDocumentSegmentUpdateApi(Resource):
     ):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
-        # check segment
-        segment_id_str = str(segment_id)
-        segment = db.session.scalar(
-            select(DocumentSegment)
-            .where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
-            .limit(1)
-        )
-        if not segment:
-            raise NotFound("Segment not found.")
         # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
         if not current_user.is_dataset_editor:
             raise Forbidden()
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
-        SegmentService.delete_segment(segment, document, dataset)
+        segment_id_str = str(segment_id)
+        _, segment = _get_segment_for_document(dataset, document, segment_id_str)
+        SegmentService.delete_segment(segment, document, dataset, db.session)
         return "", 204
 
 
@@ -576,12 +582,12 @@ class DatasetDocumentSegmentBatchImportApi(Resource):
     def post(self, current_tenant_id: str, current_user: Account, dataset_id: UUID, document_id: UUID):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
 
@@ -651,25 +657,20 @@ class ChildChunkAddApi(Resource):
     ):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
-        # check segment
-        segment_id_str = str(segment_id)
-        segment = db.session.scalar(
-            select(DocumentSegment)
-            .where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
-            .limit(1)
-        )
-        if not segment:
-            raise NotFound("Segment not found.")
         if not current_user.is_dataset_editor:
             raise Forbidden()
+        try:
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
+        except services.errors.account.NoPermissionError as e:
+            raise Forbidden(str(e))
         # check embedding model setting
         if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
             try:
@@ -686,14 +687,12 @@ class ChildChunkAddApi(Resource):
                 )
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
-        try:
-            DatasetService.check_dataset_permission(dataset, current_user)
-        except services.errors.account.NoPermissionError as e:
-            raise Forbidden(str(e))
+        segment_id_str = str(segment_id)
+        _, segment = _get_segment_for_document(dataset, document, segment_id_str)
         # validate args
         try:
             payload = ChildChunkCreatePayload.model_validate(console_ns.payload or {})
-            child_chunk = SegmentService.create_child_chunk(payload.content, segment, document, dataset)
+            child_chunk = SegmentService.create_child_chunk(payload.content, segment, document, dataset, db.session)
         except ChildChunkIndexingServiceError as e:
             raise ChildChunkIndexingError(str(e))
         return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200
@@ -709,25 +708,18 @@ class ChildChunkAddApi(Resource):
     def get(self, current_tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
-        # check segment
         segment_id_str = str(segment_id)
-        segment = db.session.scalar(
-            select(DocumentSegment)
-            .where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
-            .limit(1)
-        )
-        if not segment:
-            raise NotFound("Segment not found.")
+        _get_segment_for_document(dataset, document, segment_id_str)
         args = query_params_from_request(ChildChunkListQuery, use_defaults_for_malformed_ints=True)
 
         page = args.page
@@ -766,36 +758,29 @@ class ChildChunkAddApi(Resource):
     ):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
-            # check segment
-        segment_id_str = str(segment_id)
-        segment = db.session.scalar(
-            select(DocumentSegment)
-            .where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
-            .limit(1)
-        )
-        if not segment:
-            raise NotFound("Segment not found.")
         # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
         if not current_user.is_dataset_editor:
             raise Forbidden()
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
+        segment_id_str = str(segment_id)
+        _, segment = _get_segment_for_document(dataset, document, segment_id_str)
         # validate args
         payload = ChildChunkBatchUpdatePayload.model_validate(console_ns.payload or {})
         try:
-            child_chunks = SegmentService.update_child_chunks(payload.chunks, segment, document, dataset)
+            child_chunks = SegmentService.update_child_chunks(payload.chunks, segment, document, dataset, db.session)
         except ChildChunkIndexingServiceError as e:
             raise ChildChunkIndexingError(str(e))
         return dump_response(ChildChunkBatchUpdateResponse, {"data": child_chunks}), 200
@@ -825,48 +810,31 @@ class ChildChunkUpdateApi(Resource):
     ):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
-        # check segment
-        segment_id_str = str(segment_id)
-        segment = db.session.scalar(
-            select(DocumentSegment)
-            .where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
-            .limit(1)
-        )
-        if not segment:
-            raise NotFound("Segment not found.")
-        # check child chunk
-        child_chunk_id_str = str(child_chunk_id)
-        child_chunk = db.session.scalar(
-            select(ChildChunk)
-            .where(
-                ChildChunk.id == child_chunk_id_str,
-                ChildChunk.tenant_id == current_tenant_id,
-                ChildChunk.segment_id == segment.id,
-                ChildChunk.document_id == document_id_str,
-            )
-            .limit(1)
-        )
-        if not child_chunk:
-            raise NotFound("Child chunk not found.")
         # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
         if not current_user.is_dataset_editor:
             raise Forbidden()
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
+        segment_id_str = str(segment_id)
+        segment_ref, _ = _get_segment_for_document(dataset, document, segment_id_str)
+        child_chunk_id_str = str(child_chunk_id)
+        child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref)
+        if not child_chunk:
+            raise NotFound("Child chunk not found.")
         try:
-            SegmentService.delete_child_chunk(child_chunk, dataset)
+            SegmentService.delete_child_chunk(child_chunk, dataset, db.session)
         except ChildChunkDeleteIndexServiceError as e:
             raise ChildChunkDeleteIndexError(str(e))
         return "", 204
@@ -893,50 +861,35 @@ class ChildChunkUpdateApi(Resource):
     ):
         # check dataset
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if not dataset:
             raise NotFound("Dataset not found.")
         # check user's model setting
         DatasetService.check_dataset_model_setting(dataset)
         # check document
         document_id_str = str(document_id)
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
-            # check segment
-        segment_id_str = str(segment_id)
-        segment = db.session.scalar(
-            select(DocumentSegment)
-            .where(DocumentSegment.id == segment_id_str, DocumentSegment.tenant_id == current_tenant_id)
-            .limit(1)
-        )
-        if not segment:
-            raise NotFound("Segment not found.")
-        # check child chunk
-        child_chunk_id_str = str(child_chunk_id)
-        child_chunk = db.session.scalar(
-            select(ChildChunk)
-            .where(
-                ChildChunk.id == child_chunk_id_str,
-                ChildChunk.tenant_id == current_tenant_id,
-                ChildChunk.segment_id == segment.id,
-                ChildChunk.document_id == document_id_str,
-            )
-            .limit(1)
-        )
-        if not child_chunk:
-            raise NotFound("Child chunk not found.")
         # The role of the current user in the ta table must be admin, owner, dataset_operator, or editor
         if not current_user.is_dataset_editor:
             raise Forbidden()
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
+        segment_id_str = str(segment_id)
+        segment_ref, segment = _get_segment_for_document(dataset, document, segment_id_str)
+        child_chunk_id_str = str(child_chunk_id)
+        child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref)
+        if not child_chunk:
+            raise NotFound("Child chunk not found.")
         # validate args
         try:
             payload = ChildChunkUpdatePayload.model_validate(console_ns.payload or {})
-            child_chunk = SegmentService.update_child_chunk(payload.content, child_chunk, segment, document, dataset)
+            child_chunk = SegmentService.update_child_chunk(
+                payload.content, child_chunk, segment, document, dataset, db.session
+            )
         except ChildChunkIndexingServiceError as e:
             raise ChildChunkIndexingError(str(e))
         return dump_response(ChildChunkDetailResponse, {"data": child_chunk}), 200

api/controllers/console/datasets/external.py

@@ -377,12 +377,12 @@ class ExternalKnowledgeHitTestingApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_PIPELINE_TEST)
     def post(self, current_user: Account, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 

api/controllers/console/datasets/hit_testing_base.py

@@ -85,12 +85,12 @@ class DatasetsHitTestingBase:
         dataset_id: str, current_user: Account | None = None, current_tenant_id: str | None = None
     ) -> Dataset:
         current_user, _ = resolve_account_fallback(current_user, current_tenant_id)
-        dataset = DatasetService.get_dataset(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 

api/controllers/console/datasets/metadata.py

@@ -61,10 +61,10 @@ class DatasetMetadataCreateApi(Resource):
         metadata_args = MetadataArgs.model_validate(console_ns.payload or {})
 
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         metadata = MetadataService.create_metadata(
             db.session(), dataset_id_str, metadata_args, current_user, current_tenant_id
@@ -81,7 +81,7 @@ class DatasetMetadataCreateApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_CREATE_AND_MANAGEMENT)
     def get(self, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
         metadata = MetadataService.get_dataset_metadatas(db.session(), dataset)
@@ -105,10 +105,10 @@ class DatasetMetadataApi(Resource):
 
         dataset_id_str = str(dataset_id)
         metadata_id_str = str(metadata_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         metadata = MetadataService.update_metadata_name(
             db.session(), dataset_id_str, metadata_id_str, name, current_user, current_tenant_id
@@ -125,10 +125,10 @@ class DatasetMetadataApi(Resource):
     def delete(self, current_user: Account, dataset_id: UUID, metadata_id: UUID):
         dataset_id_str = str(dataset_id)
         metadata_id_str = str(metadata_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         MetadataService.delete_metadata(db.session(), dataset_id_str, metadata_id_str)
         # Frontend callers only await success and invalidate metadata caches; no response body is consumed.
@@ -162,10 +162,10 @@ class DatasetMetadataBuiltInFieldActionApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_EDIT)
     def post(self, current_user: Account, dataset_id: UUID, action: Literal["enable", "disable"]):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         match action:
             case "enable":
@@ -191,10 +191,10 @@ class DocumentMetadataEditApi(Resource):
     @rbac_permission_required(RBACResourceScope.DATASET, RBACPermission.DATASET_EDIT)
     def post(self, current_user: Account, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         metadata_args = MetadataOperationData.model_validate(console_ns.payload or {})
 

api/controllers/console/datasets/rag_pipeline/rag_pipeline_datasets.py

@@ -1,6 +1,5 @@
 from flask_restx import Resource
 from pydantic import BaseModel
-from sqlalchemy.orm import Session
 from werkzeug.exceptions import Forbidden
 
 import services
@@ -66,19 +65,19 @@ class CreateRagPipelineDatasetApi(Resource):
             yaml_content=payload.yaml_content,
         )
         try:
-            with Session(db.engine, expire_on_commit=False) as session:
-                rag_pipeline_dsl_service = RagPipelineDslService(session)
-                import_info = rag_pipeline_dsl_service.create_rag_pipeline_dataset(
-                    tenant_id=current_tenant_id,
-                    rag_pipeline_dataset_create_entity=rag_pipeline_dataset_create_entity,
-                )
-                session.commit()
+            rag_pipeline_dsl_service = RagPipelineDslService(db.session)
+            import_info = rag_pipeline_dsl_service.create_rag_pipeline_dataset(
+                tenant_id=current_tenant_id,
+                rag_pipeline_dataset_create_entity=rag_pipeline_dataset_create_entity,
+            )
             if rag_pipeline_dataset_create_entity.permission == "partial_members":
                 DatasetPermissionService.update_partial_member_list(
                     current_tenant_id,
                     import_info["dataset_id"],
                     rag_pipeline_dataset_create_entity.partial_member_list,
+                    db.session,
                 )
+            db.session.commit()
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()
 
@@ -111,5 +110,6 @@ class CreateEmptyRagPipelineDatasetApi(Resource):
                 permission=DatasetPermissionEnum.ONLY_ME,
                 partial_member_list=None,
             ),
+            session=db.session,
         )
         return dump_response(DatasetDetailResponse, dataset), 201

api/controllers/console/datasets/rag_pipeline/rag_pipeline_workflow.py

@@ -64,6 +64,7 @@ from services.rag_pipeline.pipeline_generate_service import PipelineGenerateServ
 from services.rag_pipeline.rag_pipeline import RagPipelineService
 from services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService
 from services.rag_pipeline.rag_pipeline_transform_service import RagPipelineTransformService
+from services.workflow_ref_service import WorkflowRefService
 from services.workflow_service import DraftWorkflowDeletionError, WorkflowInUseError, WorkflowService
 
 logger = logging.getLogger(__name__)
@@ -738,15 +739,15 @@ class RagPipelineByIdApi(Resource):
             return {"message": "No valid fields to update"}, 400
 
         rag_pipeline_service = RagPipelineService()
+        workflow_ref = WorkflowRefService.create_pipeline_workflow_ref(pipeline, workflow_id)
 
         # Create a session and manage the transaction
         with sessionmaker(db.engine, expire_on_commit=False).begin() as session:
             workflow = rag_pipeline_service.update_workflow(
                 session=session,
-                workflow_id=workflow_id,
-                tenant_id=pipeline.tenant_id,
                 account_id=current_user.id,
                 data=update_data,
+                workflow_ref=workflow_ref,
             )
 
             if not workflow:
@@ -769,13 +770,13 @@ class RagPipelineByIdApi(Resource):
             abort(400, description=f"Cannot delete workflow that is currently in use by pipeline '{pipeline.id}'")
 
         workflow_service = WorkflowService()
+        workflow_ref = WorkflowRefService.create_pipeline_workflow_ref(pipeline, workflow_id)
 
         with sessionmaker(db.engine).begin() as session:
             try:
                 workflow_service.delete_workflow(
                     session=session,
-                    workflow_id=workflow_id,
-                    tenant_id=pipeline.tenant_id,
+                    workflow_ref=workflow_ref,
                 )
             except WorkflowInUseError as e:
                 abort(400, description=str(e))

api/controllers/console/explore/audio.py

@@ -22,7 +22,9 @@ from controllers.console.explore.wraps import InstalledAppResource
 from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotInitError, QuotaExceededError
 from extensions.ext_database import db
 from graphon.model_runtime.errors.invoke import InvokeError
+from libs.login import current_account_with_tenant
 from models.model import InstalledApp
+from services.app_ref_service import AppRefService
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,
@@ -99,13 +101,22 @@ class ChatTextApi(InstalledAppResource):
             message_id = payload.message_id
             text = payload.text
             voice = payload.voice
+            message_ref = None
+            if message_id:
+                current_user, _ = current_account_with_tenant()
+                app_ref = AppRefService.create_app_ref(app_model)
+                message_ref = AppRefService.create_message_ref(
+                    app_ref,
+                    message_id,
+                    account_id=current_user.id,
+                )
 
             response = AudioService.transcript_tts(
                 app_model=app_model,
                 session=db.session,
                 text=text,
                 voice=voice,
-                message_id=message_id,
+                message_ref=message_ref,
             )
             return response
         except services.errors.app_model_config.AppModelConfigBrokenError:

api/controllers/console/explore/recommended_app.py

@@ -3,7 +3,7 @@ from uuid import UUID
 
 from flask import request
 from flask_restx import Resource
-from pydantic import BaseModel, Field, RootModel, computed_field, field_validator
+from pydantic import BaseModel, Field, computed_field, field_validator
 
 from constants.languages import languages
 from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
@@ -70,8 +70,14 @@ class LearnDifyAppListResponse(ResponseModel):
     recommended_apps: list[RecommendedAppResponse]
 
 
-class RecommendedAppDetailResponse(RootModel[dict[str, Any]]):
-    root: dict[str, Any]
+class RecommendedAppDetailResponse(ResponseModel):
+    id: str
+    name: str
+    icon: str | None = None
+    icon_background: str | None = None
+    mode: str
+    export_data: str
+    can_trial: bool | None = None
 
 
 register_schema_models(

api/controllers/console/explore/trial.py

@@ -1,9 +1,10 @@
 import logging
+from datetime import datetime
 from typing import Any, Literal, cast
 
 from flask import request
 from flask_restx import Resource, fields, marshal, marshal_with
-from pydantic import BaseModel, Field
+from pydantic import AliasChoices, BaseModel, Field, field_validator
 from sqlalchemy import select
 from werkzeug.exceptions import Forbidden, InternalServerError, NotFound
 
@@ -63,6 +64,7 @@ from fields.app_fields import (
     site_fields,
     tag_fields,
 )
+from fields.base import ResponseModel
 from fields.dataset_fields import dataset_fields
 from fields.member_fields import simple_account_fields
 from fields.message_fields import SuggestedQuestionsResponse
@@ -75,12 +77,13 @@ from fields.workflow_fields import (
 from graphon.graph_engine.manager import GraphEngineManager
 from graphon.model_runtime.errors.invoke import InvokeError
 from libs import helper
-from libs.helper import uuid_value
+from libs.helper import to_timestamp, uuid_value
 from models import Account
 from models.account import TenantStatus
 from models.model import AppMode, Site
 from models.workflow import Workflow
 from services.app_generate_service import AppGenerateService
+from services.app_ref_service import AppRefService
 from services.app_service import AppService
 from services.audio_service import AudioService
 from services.dataset_service import DatasetService
@@ -179,6 +182,254 @@ class TrialDatasetListQuery(BaseModel):
     ids: list[str] = Field(default_factory=list, description="Dataset IDs")
 
 
+type TrialAppMode = Literal["chat", "agent-chat", "advanced-chat", "workflow", "completion"]
+type TrialIconType = Literal["emoji", "image", "link"]
+type JsonObject = dict[str, Any]
+
+
+class TrialAppModel(ResponseModel):
+    provider: str
+    name: str
+    mode: str | None = None
+    completion_params: JsonObject = Field(default_factory=dict)
+
+
+class TrialAppAgentMode(ResponseModel):
+    enabled: bool | None = None
+    strategy: str | None = None
+    tools: list[JsonObject] = Field(default_factory=list)
+
+
+class TrialAppModelConfigResponse(ResponseModel):
+    opening_statement: str | None = None
+    suggested_questions: list[str] = Field(
+        default_factory=list,
+        validation_alias=AliasChoices("suggested_questions_list", "suggested_questions"),
+    )
+    suggested_questions_after_answer: JsonObject | None = Field(
+        default=None,
+        validation_alias=AliasChoices("suggested_questions_after_answer_dict", "suggested_questions_after_answer"),
+    )
+    speech_to_text: JsonObject | None = Field(
+        default=None, validation_alias=AliasChoices("speech_to_text_dict", "speech_to_text")
+    )
+    text_to_speech: JsonObject | None = Field(
+        default=None, validation_alias=AliasChoices("text_to_speech_dict", "text_to_speech")
+    )
+    retriever_resource: JsonObject | None = Field(
+        default=None, validation_alias=AliasChoices("retriever_resource_dict", "retriever_resource")
+    )
+    annotation_reply: JsonObject | None = Field(
+        default=None, validation_alias=AliasChoices("annotation_reply_dict", "annotation_reply")
+    )
+    more_like_this: JsonObject | None = Field(
+        default=None, validation_alias=AliasChoices("more_like_this_dict", "more_like_this")
+    )
+    sensitive_word_avoidance: JsonObject | None = Field(
+        default=None, validation_alias=AliasChoices("sensitive_word_avoidance_dict", "sensitive_word_avoidance")
+    )
+    external_data_tools: list[JsonObject] = Field(
+        default_factory=list, validation_alias=AliasChoices("external_data_tools_list", "external_data_tools")
+    )
+    model: TrialAppModel | None = Field(default=None, validation_alias=AliasChoices("model_dict", "model"))
+    user_input_form: list[JsonObject] = Field(
+        default_factory=list, validation_alias=AliasChoices("user_input_form_list", "user_input_form")
+    )
+    dataset_query_variable: str | None = None
+    pre_prompt: str | None = None
+    agent_mode: TrialAppAgentMode | None = Field(
+        default=None,
+        validation_alias=AliasChoices("agent_mode_dict", "agent_mode"),
+    )
+    prompt_type: str | None = None
+    chat_prompt_config: JsonObject | None = Field(
+        default=None, validation_alias=AliasChoices("chat_prompt_config_dict", "chat_prompt_config")
+    )
+    completion_prompt_config: JsonObject | None = Field(
+        default=None, validation_alias=AliasChoices("completion_prompt_config_dict", "completion_prompt_config")
+    )
+    dataset_configs: JsonObject | None = Field(
+        default=None,
+        validation_alias=AliasChoices("dataset_configs_dict", "dataset_configs"),
+    )
+    file_upload: JsonObject | None = Field(
+        default=None,
+        validation_alias=AliasChoices("file_upload_dict", "file_upload"),
+    )
+    created_by: str | None = None
+    created_at: int | None = None
+    updated_by: str | None = None
+    updated_at: int | None = None
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
+        return to_timestamp(value)
+
+
+class TrialDeletedToolResponse(ResponseModel):
+    type: str
+    tool_name: str
+    provider_id: str
+
+
+class TrialTagResponse(ResponseModel):
+    id: str
+    name: str
+    type: str
+
+
+class TrialSiteResponse(ResponseModel):
+    access_token: str | None = Field(default=None, validation_alias="code")
+    code: str | None = None
+    title: str
+    icon_type: TrialIconType | None = None
+    icon: str | None = None
+    icon_background: str | None = None
+    description: str | None = None
+    default_language: str
+    chat_color_theme: str | None = None
+    chat_color_theme_inverted: bool | None = None
+    customize_domain: str | None = None
+    copyright: str | None = None
+    privacy_policy: str | None = None
+    input_placeholder: str | None = None
+    custom_disclaimer: str | None = None
+    customize_token_strategy: str | None = None
+    prompt_public: bool | None = None
+    app_base_url: str | None = None
+    show_workflow_steps: bool | None = None
+    use_icon_as_answer_icon: bool | None = None
+    created_by: str | None = None
+    created_at: int | None = None
+    updated_by: str | None = None
+    updated_at: int | None = None
+    icon_url: str | None = None
+
+    @field_validator("icon_type", mode="before")
+    @classmethod
+    def _normalize_icon_type(cls, value: Any) -> str | None:
+        if hasattr(value, "value"):
+            return value.value
+        return value
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
+        return to_timestamp(value)
+
+
+class TrialWorkflowPartialResponse(ResponseModel):
+    id: str
+    created_by: str | None = None
+    created_at: int | None = None
+    updated_by: str | None = None
+    updated_at: int | None = None
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
+        return to_timestamp(value)
+
+
+class TrialAppDetailResponse(ResponseModel):
+    id: str
+    name: str
+    description: str | None = None
+    mode: TrialAppMode = Field(validation_alias="mode_compatible_with_agent")
+    icon_type: TrialIconType | None = None
+    icon: str | None = None
+    icon_background: str | None = None
+    icon_url: str | None = None
+    enable_site: bool
+    enable_api: bool
+    model_config_: TrialAppModelConfigResponse | None = Field(
+        default=None,
+        validation_alias=AliasChoices("app_model_config", "model_config"),
+        alias="model_config",
+    )
+    workflow: TrialWorkflowPartialResponse | None = None
+    api_base_url: str | None = None
+    use_icon_as_answer_icon: bool | None = None
+    max_active_requests: int | None = None
+    created_by: str | None = None
+    created_at: int | None = None
+    updated_by: str | None = None
+    updated_at: int | None = None
+    deleted_tools: list[TrialDeletedToolResponse] = Field(default_factory=list)
+    access_mode: str | None = None
+    tags: list[TrialTagResponse] = Field(default_factory=list)
+    permission_keys: list[str] = Field(default_factory=list)
+    site: TrialSiteResponse
+
+    @field_validator("icon_type", mode="before")
+    @classmethod
+    def _normalize_icon_type(cls, value: Any) -> str | None:
+        if hasattr(value, "value"):
+            return value.value
+        return value
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
+        return to_timestamp(value)
+
+
+class TrialDatasetResponse(ResponseModel):
+    id: str
+    name: str
+    description: str | None = None
+    permission: str | None = None
+    data_source_type: str | None = None
+    indexing_technique: str | None = None
+    created_by: str | None = None
+    created_at: int | None = None
+    permission_keys: list[str] = Field(default_factory=list)
+
+
+class TrialDatasetListResponse(ResponseModel):
+    data: list[TrialDatasetResponse]
+    has_more: bool
+    limit: int
+    total: int
+    page: int
+
+
+class TrialWorkflowAccount(ResponseModel):
+    id: str
+    name: str | None = None
+    email: str | None = None
+
+
+class TrialWorkflowResponse(ResponseModel):
+    id: str
+    graph: JsonObject = Field(validation_alias=AliasChoices("graph_dict", "graph"))
+    features: JsonObject = Field(default_factory=dict, validation_alias=AliasChoices("features_dict", "features"))
+    hash: str | None = Field(default=None, validation_alias=AliasChoices("unique_hash", "hash"))
+    version: str | None = None
+    marked_name: str | None = None
+    marked_comment: str | None = None
+    created_by: TrialWorkflowAccount | None = Field(
+        default=None,
+        validation_alias=AliasChoices("created_by_account", "created_by"),
+    )
+    created_at: int | None = None
+    updated_by: TrialWorkflowAccount | None = Field(
+        default=None,
+        validation_alias=AliasChoices("updated_by_account", "updated_by"),
+    )
+    updated_at: int | None = None
+    tool_published: bool | None = None
+    environment_variables: list[JsonObject] = Field(default_factory=list)
+    conversation_variables: list[JsonObject] = Field(default_factory=list)
+    rag_pipeline_variables: list[JsonObject] = Field(default_factory=list)
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
+        return to_timestamp(value)
+
+
 register_schema_models(
     console_ns,
     WorkflowRunRequest,
@@ -196,6 +447,9 @@ register_response_schema_models(
     SimpleResultResponse,
     SiteResponse,
     SuggestedQuestionsResponse,
+    TrialAppDetailResponse,
+    TrialDatasetListResponse,
+    TrialWorkflowResponse,
 )
 
 
@@ -414,6 +668,14 @@ class TrialChatTextApi(TrialAppResource):
             message_id = request_data.message_id
             text = request_data.text
             voice = request_data.voice
+            message_ref = None
+            if message_id:
+                app_ref = AppRefService.create_app_ref(app_model)
+                message_ref = AppRefService.create_message_ref(
+                    app_ref,
+                    message_id,
+                    account_id=current_user.id,
+                )
 
             # Get IDs before they might be detached from session
             app_id = app_model.id
@@ -424,7 +686,7 @@ class TrialChatTextApi(TrialAppResource):
                 session=db.session,
                 text=text,
                 voice=voice,
-                message_id=message_id,
+                message_ref=message_ref,
             )
             RecommendedAppService.add_trial_app_record(db.session, app_id, user_id)
             return response
@@ -557,7 +819,7 @@ class TrialAppParameterApi(Resource):
 
 
 class AppApi(Resource):
-    @console_ns.response(200, "Success", app_detail_with_site_model)
+    @console_ns.response(200, "Success", console_ns.models[TrialAppDetailResponse.__name__])
     @get_app_model_with_trial(None)
     @marshal_with(app_detail_with_site_model)
     def get(self, app_model):
@@ -570,7 +832,7 @@ class AppApi(Resource):
 
 
 class AppWorkflowApi(Resource):
-    @console_ns.response(200, "Success", workflow_model)
+    @console_ns.response(200, "Success", console_ns.models[TrialWorkflowResponse.__name__])
     @get_app_model_with_trial(None)
     @marshal_with(workflow_model)
     def get(self, app_model):
@@ -584,7 +846,7 @@ class AppWorkflowApi(Resource):
 
 class DatasetListApi(Resource):
     @console_ns.doc(params=query_params_from_model(TrialDatasetListQuery))
-    @console_ns.response(200, "Success", dataset_list_model)
+    @console_ns.response(200, "Success", console_ns.models[TrialDatasetListResponse.__name__])
     @get_app_model_with_trial(None)
     def get(self, app_model):
         page = request.args.get("page", default=1, type=int)

api/controllers/console/files.py

@@ -38,6 +38,22 @@ register_response_schema_models(console_ns, AllowedExtensionsResponse, TextConte
 
 PREVIEW_WORDS_LIMIT = 3000
 
+_FILE_UPLOAD_PARAMS = {
+    "file": {
+        "description": "File to upload",
+        "in": "formData",
+        "type": "file",
+        "required": True,
+    },
+    "source": {
+        "description": "Optional upload source",
+        "in": "formData",
+        "type": "string",
+        "enum": ["datasets"],
+        "required": False,
+    },
+}
+
 
 @console_ns.route("/files/upload")
 class FileApi(Resource):
@@ -64,6 +80,7 @@ class FileApi(Resource):
     @login_required
     @account_initialization_required
     @cloud_edition_billing_resource_check("documents")
+    @console_ns.doc(consumes=["multipart/form-data"], params=_FILE_UPLOAD_PARAMS)
     @console_ns.response(201, "File uploaded successfully", console_ns.models[FileResponse.__name__])
     @with_current_user
     def post(self, current_user: Account):

api/controllers/console/snippets/snippet_workflow.py

@@ -8,6 +8,7 @@ from pydantic import BaseModel, Field
 from sqlalchemy.orm import Session, sessionmaker
 from werkzeug.exceptions import BadRequest, InternalServerError, NotFound
 
+from controllers.common.controller_schemas import WorkflowUpdatePayload
 from controllers.common.fields import GeneratedAppResponse, SimpleResultResponse
 from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
 from controllers.console import console_ns
@@ -96,6 +97,7 @@ register_schema_models(
     SnippetLoopNodeRunPayload,
     SnippetWorkflowListQuery,
     WorkflowRunQuery,
+    WorkflowUpdatePayload,
     PublishWorkflowPayload,
 )
 register_response_schema_models(
@@ -165,7 +167,6 @@ class SnippetDraftWorkflowApi(Resource):
     @account_initialization_required
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_MANAGE, resource_required=False)
     def get(self, snippet: CustomizedSnippet):
         """Get draft workflow for snippet."""
         snippet_service = _snippet_service()
@@ -234,7 +235,6 @@ class SnippetDraftConfigApi(Resource):
     @account_initialization_required
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_MANAGE, resource_required=False)
     def get(self, snippet: CustomizedSnippet):
         """Get snippet draft workflow configuration limits."""
         return {
@@ -256,7 +256,6 @@ class SnippetPublishedWorkflowApi(Resource):
     @account_initialization_required
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_MANAGE, resource_required=False)
     def get(self, snippet: CustomizedSnippet):
         """Get published workflow for snippet."""
         if not snippet.is_published:
@@ -321,7 +320,6 @@ class SnippetDefaultBlockConfigsApi(Resource):
     @account_initialization_required
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_MANAGE, resource_required=False)
     def get(self, snippet: CustomizedSnippet):
         """Get default block configurations for snippet workflow."""
         snippet_service = _snippet_service()
@@ -344,7 +342,9 @@ class SnippetPublishedAllWorkflowApi(Resource):
     @account_initialization_required
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_MANAGE, resource_required=False)
+    @rbac_permission_required(
+        RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
+    )
     def get(self, snippet: CustomizedSnippet):
         """Get all published workflow versions for snippet."""
         args = SnippetWorkflowListQuery.model_validate(request.args.to_dict(flat=True))
@@ -413,6 +413,49 @@ class SnippetDraftWorkflowRestoreApi(Resource):
         }
 
 
+@console_ns.route("/snippets/<uuid:snippet_id>/workflows/<string:workflow_id>")
+class SnippetWorkflowByIdApi(Resource):
+    @console_ns.doc("update_snippet_workflow_by_id")
+    @console_ns.doc(description="Update published snippet workflow attributes")
+    @console_ns.doc(params={"snippet_id": "Snippet ID", "workflow_id": "Workflow ID"})
+    @console_ns.expect(console_ns.models[WorkflowUpdatePayload.__name__])
+    @console_ns.response(200, "Workflow updated successfully", console_ns.models[SnippetWorkflowResponse.__name__])
+    @console_ns.response(400, "No valid fields to update")
+    @console_ns.response(404, "Workflow not found")
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @with_current_user
+    @get_snippet
+    @edit_permission_required
+    @rbac_permission_required(
+        RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
+    )
+    def patch(self, current_user: Account, snippet: CustomizedSnippet, workflow_id: str):
+        """Update a published snippet workflow version's display metadata."""
+        payload = WorkflowUpdatePayload.model_validate(console_ns.payload or {})
+        update_data = payload.model_dump(exclude_unset=True)
+
+        if not update_data:
+            return {"message": "No valid fields to update"}, 400
+
+        snippet_service = _snippet_service()
+        with _snippet_session_maker().begin() as session:
+            workflow = snippet_service.update_workflow(
+                session=session,
+                snippet=snippet,
+                workflow_id=workflow_id,
+                account=current_user,
+                data=update_data,
+            )
+            if not workflow:
+                raise NotFound("Workflow not found")
+
+        response = SnippetWorkflowResponse.model_validate(workflow, from_attributes=True).model_dump(mode="json")
+        response["input_fields"] = snippet.input_fields_list
+        return response
+
+
 @console_ns.route("/snippets/<uuid:snippet_id>/workflow-runs")
 class SnippetWorkflowRunsApi(Resource):
     @console_ns.doc("list_snippet_workflow_runs")
@@ -514,9 +557,6 @@ class SnippetDraftNodeRunApi(Resource):
     @with_current_user
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(
-        RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
-    )
     def post(self, current_user: Account, snippet: CustomizedSnippet, node_id: str):
         """
         Run a single node in snippet draft workflow.
@@ -605,9 +645,6 @@ class SnippetDraftRunIterationNodeApi(Resource):
     @with_current_user
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(
-        RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
-    )
     def post(self, current_user: Account, snippet: CustomizedSnippet, node_id: str):
         """
         Run a draft workflow iteration node for snippet.
@@ -653,9 +690,6 @@ class SnippetDraftRunLoopNodeApi(Resource):
     @with_current_user
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(
-        RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
-    )
     def post(self, current_user: Account, snippet: CustomizedSnippet, node_id: str):
         """
         Run a draft workflow loop node for snippet.
@@ -699,9 +733,6 @@ class SnippetDraftWorkflowRunApi(Resource):
     @with_current_user
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(
-        RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
-    )
     def post(self, current_user: Account, snippet: CustomizedSnippet):
         """
         Run draft workflow for snippet.
@@ -740,9 +771,6 @@ class SnippetWorkflowTaskStopApi(Resource):
     @account_initialization_required
     @get_snippet
     @edit_permission_required
-    @rbac_permission_required(
-        RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
-    )
     def post(self, snippet: CustomizedSnippet, task_id: str):
         """
         Stop a running snippet workflow task.

api/controllers/console/snippets/snippet_workflow_draft_variable.py

@@ -34,11 +34,8 @@ from controllers.console.app.workflow_draft_variable import (
 )
 from controllers.console.snippets.snippet_workflow import get_snippet
 from controllers.console.wraps import (
-    RBACPermission,
-    RBACResourceScope,
     account_initialization_required,
     edit_permission_required,
-    rbac_permission_required,
     setup_required,
     with_current_user,
 )
@@ -105,7 +102,6 @@ class SnippetWorkflowVariableCollectionApi(Resource):
     )
     @_snippet_draft_var_prerequisite
     @marshal_with(workflow_draft_variable_list_without_value_model)
-    @rbac_permission_required(RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_MANAGE, resource_required=False)
     def get(self, current_user: Account, snippet: CustomizedSnippet) -> WorkflowDraftVariableList:
         args = WorkflowDraftVariableListQuery.model_validate(request.args.to_dict(flat=True))  # type: ignore
 
@@ -129,9 +125,6 @@ class SnippetWorkflowVariableCollectionApi(Resource):
     @console_ns.doc(description="Delete all draft workflow variables for the current user (snippet scope)")
     @console_ns.response(204, "Workflow variables deleted successfully")
     @_snippet_draft_var_prerequisite
-    @rbac_permission_required(
-        RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
-    )
     def delete(self, current_user: Account, snippet: CustomizedSnippet) -> Response:
         draft_var_srv = WorkflowDraftVariableService(session=db.session())
         draft_var_srv.delete_user_workflow_variables(snippet.id, user_id=current_user.id)

api/controllers/console/tag/tags.py

@@ -4,12 +4,17 @@ from uuid import UUID
 from flask import request
 from flask_restx import Resource
 from pydantic import BaseModel, Field, field_validator
+from sqlalchemy import select
 from werkzeug.exceptions import Forbidden
 
+from configs import dify_config
 from controllers.common.fields import SimpleResultResponse
 from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
+from controllers.common.wraps import enforce_rbac_access
 from controllers.console import console_ns
 from controllers.console.wraps import (
+    RBACPermission,
+    RBACResourceScope,
     account_initialization_required,
     edit_permission_required,
     setup_required,
@@ -18,9 +23,10 @@ from controllers.console.wraps import (
 )
 from extensions.ext_database import db
 from fields.base import ResponseModel
-from libs.login import login_required
+from libs.login import current_account_with_tenant, login_required
 from models import Account
 from models.enums import TagType
+from models.model import Tag
 from services.tag_service import (
     SaveTagPayload,
     TagBindingCreatePayload,
@@ -91,6 +97,31 @@ register_schema_models(
 register_response_schema_models(console_ns, SimpleResultResponse)
 
 
+def _enforce_snippet_tag_rbac_if_needed(tag_type: TagType | str | None) -> None:
+    if tag_type != TagType.SNIPPET:
+        return
+    if not dify_config.RBAC_ENABLED:
+        return
+
+    current_user, current_tenant_id = current_account_with_tenant()
+    enforce_rbac_access(
+        tenant_id=current_tenant_id,
+        account_id=current_user.id,
+        resource_type=RBACResourceScope.WORKSPACE,
+        scene=RBACPermission.SNIPPETS_CREATE_AND_MODIFY,
+        resource_required=False,
+    )
+
+
+def _enforce_snippet_tag_rbac_by_tag_id(tag_id: str) -> None:
+    if not dify_config.RBAC_ENABLED:
+        return
+
+    _, current_tenant_id = current_account_with_tenant()
+    tag_type = db.session.scalar(select(Tag.type).where(Tag.id == tag_id, Tag.tenant_id == current_tenant_id).limit(1))
+    _enforce_snippet_tag_rbac_if_needed(tag_type)
+
+
 @console_ns.route("/tags")
 class TagListApi(Resource):
     @setup_required
@@ -122,6 +153,7 @@ class TagListApi(Resource):
             raise Forbidden()
 
         payload = TagBasePayload.model_validate(console_ns.payload or {})
+        _enforce_snippet_tag_rbac_if_needed(payload.type)
         tag = TagService.save_tags(SaveTagPayload(name=payload.name, type=payload.type), db.session)
 
         response = TagResponse.model_validate(
@@ -146,6 +178,7 @@ class TagUpdateDeleteApi(Resource):
             raise Forbidden()
 
         payload = TagUpdateRequestPayload.model_validate(console_ns.payload or {})
+        _enforce_snippet_tag_rbac_by_tag_id(tag_id_str)
         tag = TagService.update_tags(UpdateTagPayload(name=payload.name), tag_id_str, db.session)
 
         binding_count = TagService.get_tag_binding_count(tag_id_str, db.session)
@@ -164,6 +197,7 @@ class TagUpdateDeleteApi(Resource):
     def delete(self, tag_id: UUID):
         tag_id_str = str(tag_id)
 
+        _enforce_snippet_tag_rbac_by_tag_id(tag_id_str)
         TagService.delete_tag(tag_id_str, db.session)
 
         return "", 204
@@ -184,6 +218,7 @@ def _create_tag_bindings(current_user: Account) -> tuple[dict[str, str], int]:
     _require_tag_binding_edit_permission(current_user)
 
     payload = TagBindingPayload.model_validate(console_ns.payload or {})
+    _enforce_snippet_tag_rbac_if_needed(payload.type)
     TagService.save_tag_binding(
         TagBindingCreatePayload(
             tag_ids=payload.tag_ids,
@@ -199,6 +234,7 @@ def _remove_tag_bindings(current_user: Account) -> tuple[dict[str, str], int]:
     _require_tag_binding_edit_permission(current_user)
 
     payload = TagBindingRemovePayload.model_validate(console_ns.payload or {})
+    _enforce_snippet_tag_rbac_if_needed(payload.type)
     TagService.delete_tag_binding(
         TagBindingDeletePayload(
             tag_ids=payload.tag_ids,

api/controllers/console/workspace/members.py

@@ -84,8 +84,6 @@ class MemberActionTenantResponse(ResponseModel):
 register_enum_models(console_ns, TenantAccountRole)
 register_schema_models(
     console_ns,
-    AccountWithRole,
-    AccountWithRoleList,
     MemberInvitePayload,
     MemberRoleUpdatePayload,
     OwnerTransferEmailPayload,
@@ -94,6 +92,8 @@ register_schema_models(
 )
 register_response_schema_models(
     console_ns,
+    AccountWithRole,
+    AccountWithRoleList,
     SimpleResultDataResponse,
     SimpleResultResponse,
     VerificationTokenResponse,

api/controllers/console/workspace/model_providers.py

@@ -18,6 +18,7 @@ from controllers.console.wraps import (
     with_current_tenant_id,
     with_current_user,
 )
+from extensions.ext_database import db
 from fields.base import ResponseModel
 from graphon.model_runtime.entities.model_entities import ModelType
 from graphon.model_runtime.errors.validate import CredentialsValidateFailedError
@@ -352,7 +353,7 @@ class ModelProviderPaymentCheckoutUrlApi(Resource):
     def get(self, current_tenant_id: str, current_user: Account, provider: str):
         if provider != "anthropic":
             raise ValueError(f"provider name {provider} is invalid")
-        BillingService.is_tenant_owner_or_admin(current_user)
+        BillingService.is_tenant_owner_or_admin(db.session, current_user)
         data = BillingService.get_model_provider_payment_link(
             provider_name=provider,
             tenant_id=current_tenant_id,

api/controllers/console/workspace/plugin.py

@@ -302,11 +302,28 @@ class PluginListResponse(ResponseModel):
 
 
 class PluginVersionsResponse(ResponseModel):
-    versions: Any
+    versions: Mapping[str, PluginService.LatestPluginCache | None]
+
+
+class PluginInstallationItemResponse(ResponseModel):
+    id: str
+    created_at: datetime
+    updated_at: datetime
+    tenant_id: str
+    endpoints_setups: int
+    endpoints_active: int
+    runtime_type: str
+    source: PluginInstallationSource
+    meta: Mapping[str, Any]
+    plugin_id: str
+    plugin_unique_identifier: str
+    version: str
+    checksum: str
+    declaration: PluginDeclarationResponse
 
 
 class PluginInstallationsResponse(ResponseModel):
-    plugins: Any
+    plugins: list[PluginInstallationItemResponse]
 
 
 class PluginManifestResponse(ResponseModel):

api/controllers/console/workspace/rbac.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-from enum import StrEnum
-from typing import Any
+from typing import Any, Literal
 
 from flask import request
 from flask_restx import Resource
@@ -10,10 +9,11 @@ from sqlalchemy import select
 from werkzeug.exceptions import NotFound
 
 from configs import dify_config
-from controllers.common.schema import register_response_schema_models
+from controllers.common.schema import query_params_from_model, register_response_schema_models, register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import RBACPermission, RBACResourceScope, rbac_permission_required
 from core.db.session_factory import session_factory
+from core.rbac import RBACResourceWhitelistScope
 from libs.login import current_account_with_tenant, login_required
 from models import Account
 from services.enterprise import rbac_service as svc
@@ -511,14 +511,8 @@ class RBACAccessPolicyBindingUnlockApi(Resource):
 # ---------------------------------------------------------------------------
 
 
-class _AccessScope(StrEnum):
-    ALL = "all"
-    SPECIFIC = "specific"
-    ONLY_ME = "only_me"
-
-
 class _ResourceAccessScopeRequest(BaseModel):
-    scope: _AccessScope
+    scope: RBACResourceWhitelistScope
 
 
 class _ReplaceBindingsRequest(BaseModel):
@@ -544,6 +538,20 @@ class _DeleteMemberBindingsRequest(BaseModel):
         return value
 
 
+class _AccessControlLanguageQuery(BaseModel):
+    language: Literal["en", "ja", "zh"] | None = Field(default=None, description="Localized policy label language")
+
+
+register_schema_models(
+    console_ns,
+    _ResourceAccessScopeRequest,
+    _ReplaceBindingsRequest,
+    _DeleteMemberBindingsRequest,
+    _AccessControlLanguageQuery,
+    svc.ReplaceUserAccessPolicies,
+)
+
+
 @console_ns.route("/workspaces/current/rbac/my-permissions")
 class RBACMyPermissionsApi(Resource):
     @login_required
@@ -563,6 +571,7 @@ class RBACMyPermissionsApi(Resource):
 @console_ns.route("/workspaces/current/rbac/apps/<uuid:app_id>/access-policy")
 class RBACAppMatrixApi(Resource):
     @login_required
+    @console_ns.doc(params=query_params_from_model(_AccessControlLanguageQuery))
     @console_ns.response(200, "Success", console_ns.models[svc.AppAccessMatrix.__name__])
     def get(self, app_id):
         tenant_id, account_id = _current_ids()
@@ -580,6 +589,7 @@ class RBACAppWhitelistApi(Resource):
         return _dump(svc.RBACService.AppAccess.whitelist(tenant_id, account_id, str(app_id)))
 
     @login_required
+    @console_ns.expect(console_ns.models[_ResourceAccessScopeRequest.__name__])
     @console_ns.response(200, "Success", console_ns.models[svc.ResourceWhitelist.__name__])
     def put(self, app_id):
         tenant_id, account_id = _current_ids()
@@ -597,6 +607,7 @@ class RBACAppWhitelistApi(Resource):
 @console_ns.route("/workspaces/current/rbac/apps/<uuid:app_id>/user-access-policies")
 class RBACAppUserAccessPoliciesApi(Resource):
     @login_required
+    @console_ns.doc(params=query_params_from_model(_AccessControlLanguageQuery))
     @console_ns.response(200, "Success", console_ns.models[svc.ResourceUserAccessPoliciesResponse.__name__])
     def get(self, app_id):
         tenant_id, account_id = _current_ids()
@@ -608,6 +619,7 @@ class RBACAppUserAccessPoliciesApi(Resource):
 @console_ns.route("/workspaces/current/rbac/apps/<uuid:app_id>/users/<uuid:target_account_id>/access-policies")
 class RBACAppUserAccessPolicyAssignmentApi(Resource):
     @login_required
+    @console_ns.expect(console_ns.models[svc.ReplaceUserAccessPolicies.__name__])
     @console_ns.response(200, "Success", console_ns.models[svc.ReplaceUserAccessPoliciesResponse.__name__])
     def put(self, app_id, target_account_id):
         tenant_id, account_id = _current_ids()
@@ -641,6 +653,7 @@ class RBACAppMemberBindingsApi(Resource):
         return _dump(svc.RBACService.AppAccess.list_member_bindings(tenant_id, account_id, str(app_id), str(policy_id)))
 
     @login_required
+    @console_ns.expect(console_ns.models[_DeleteMemberBindingsRequest.__name__])
     @console_ns.response(200, "Success", console_ns.models[svc.MemberBindingsResponse.__name__])
     def delete(self, app_id, policy_id):
         tenant_id, account_id = _current_ids()
@@ -663,6 +676,7 @@ class RBACAppMemberBindingsApi(Resource):
 @console_ns.route("/workspaces/current/rbac/datasets/<uuid:dataset_id>/access-policy")
 class RBACDatasetMatrixApi(Resource):
     @login_required
+    @console_ns.doc(params=query_params_from_model(_AccessControlLanguageQuery))
     @console_ns.response(200, "Success", console_ns.models[svc.DatasetAccessMatrix.__name__])
     def get(self, dataset_id):
         tenant_id, account_id = _current_ids()
@@ -680,6 +694,7 @@ class RBACDatasetWhitelistApi(Resource):
         return _dump(svc.RBACService.DatasetAccess.whitelist(tenant_id, account_id, str(dataset_id)))
 
     @login_required
+    @console_ns.expect(console_ns.models[_ResourceAccessScopeRequest.__name__])
     @console_ns.response(200, "Success", console_ns.models[svc.ResourceWhitelist.__name__])
     def put(self, dataset_id):
         tenant_id, account_id = _current_ids()
@@ -697,6 +712,7 @@ class RBACDatasetWhitelistApi(Resource):
 @console_ns.route("/workspaces/current/rbac/datasets/<uuid:dataset_id>/user-access-policies")
 class RBACDatasetUserAccessPoliciesApi(Resource):
     @login_required
+    @console_ns.doc(params=query_params_from_model(_AccessControlLanguageQuery))
     @console_ns.response(200, "Success", console_ns.models[svc.ResourceUserAccessPoliciesResponse.__name__])
     def get(self, dataset_id):
         tenant_id, account_id = _current_ids()
@@ -708,6 +724,7 @@ class RBACDatasetUserAccessPoliciesApi(Resource):
 @console_ns.route("/workspaces/current/rbac/datasets/<uuid:dataset_id>/users/<uuid:target_account_id>/access-policies")
 class RBACDatasetUserAccessPolicyAssignmentApi(Resource):
     @login_required
+    @console_ns.expect(console_ns.models[svc.ReplaceUserAccessPolicies.__name__])
     @console_ns.response(200, "Success", console_ns.models[svc.ReplaceUserAccessPoliciesResponse.__name__])
     def put(self, dataset_id, target_account_id):
         tenant_id, account_id = _current_ids()
@@ -747,6 +764,7 @@ class RBACDatasetMemberBindingsApi(Resource):
         )
 
     @login_required
+    @console_ns.expect(console_ns.models[_DeleteMemberBindingsRequest.__name__])
     @console_ns.response(200, "Success", console_ns.models[svc.MemberBindingsResponse.__name__])
     def delete(self, dataset_id, policy_id):
         tenant_id, account_id = _current_ids()
@@ -785,6 +803,7 @@ class RBACWorkspaceAppRoleBindingsApi(Resource):
 @console_ns.route("/workspaces/current/rbac/workspace/apps/access-policies/<uuid:policy_id>/bindings")
 class RBACWorkspaceAppBindingsApi(Resource):
     @login_required
+    @console_ns.expect(console_ns.models[_ReplaceBindingsRequest.__name__])
     @console_ns.response(200, "Success", console_ns.models[svc.AccessMatrixItem.__name__])
     def put(self, policy_id):
         tenant_id, account_id = _current_ids()
@@ -832,6 +851,7 @@ class RBACWorkspaceDatasetRoleBindingsApi(Resource):
 @console_ns.route("/workspaces/current/rbac/workspace/datasets/access-policies/<uuid:policy_id>/bindings")
 class RBACWorkspaceDatasetBindingsApi(Resource):
     @login_required
+    @console_ns.expect(console_ns.models[_ReplaceBindingsRequest.__name__])
     @console_ns.response(200, "Success", console_ns.models[svc.AccessMatrixItem.__name__])
     def put(self, policy_id):
         tenant_id, account_id = _current_ids()
@@ -873,6 +893,9 @@ class _ReplaceMemberRolesRequest(BaseModel):
         return value
 
 
+register_schema_models(console_ns, _ReplaceMemberRolesRequest)
+
+
 @console_ns.route("/workspaces/current/rbac/members/<uuid:member_id>/rbac-roles")
 class RBACMemberRolesApi(Resource):
     @login_required
@@ -882,6 +905,7 @@ class RBACMemberRolesApi(Resource):
         return _dump(svc.RBACService.MemberRoles.get(tenant_id, account_id, str(member_id)))
 
     @login_required
+    @console_ns.expect(console_ns.models[_ReplaceMemberRolesRequest.__name__])
     @console_ns.response(200, "Success", console_ns.models[svc.MemberRolesResponse.__name__])
     def put(self, member_id):
         tenant_id, account_id = _current_ids()

api/controllers/console/workspace/snippets.py

@@ -1,11 +1,13 @@
 import logging
 import re
+from datetime import datetime
 from typing import Any
 from urllib.parse import quote
 
 from flask import Response, request
 from flask_restx import Resource, marshal
-from pydantic import RootModel
+from pydantic import Field as PydanticField
+from pydantic import field_validator
 from sqlalchemy.orm import Session, sessionmaker
 from werkzeug.datastructures import MultiDict
 from werkzeug.exceptions import NotFound
@@ -30,27 +32,34 @@ from controllers.console.wraps import (
     with_current_tenant_id,
     with_current_user,
 )
+from core.plugin.entities.plugin import PluginDependency
 from extensions.ext_database import db
 from fields.base import ResponseModel
-from fields.snippet_fields import snippet_fields, snippet_list_fields, snippet_pagination_fields
+from fields.snippet_fields import snippet_fields, snippet_list_fields
+from libs.helper import to_timestamp
 from libs.login import login_required
 from models import Account
 from models.snippet import SnippetType
-from services.app_dsl_service import ImportStatus
-from services.snippet_dsl_service import SnippetDslService
+from services.snippet_dsl_service import ImportStatus, SnippetDslService
 from services.snippet_service import SnippetService
 
 logger = logging.getLogger(__name__)
 _TAG_IDS_BRACKET_PATTERN = re.compile(r"^tag_ids\[(\d+)\]$")
 _CREATOR_IDS_BRACKET_PATTERN = re.compile(r"^creator_ids\[(\d+)\]$")
+_CREATORS_BRACKET_PATTERN = re.compile(r"^creators\[(\d+)\]$")
 
 
-class SnippetImportResponse(RootModel[dict[str, Any]]):
-    root: dict[str, Any]
+class SnippetImportResponse(ResponseModel):
+    id: str
+    status: ImportStatus
+    snippet_id: str | None
+    current_dsl_version: str
+    imported_dsl_version: str
+    error: str
 
 
-class SnippetDependencyCheckResponse(RootModel[dict[str, Any]]):
-    root: dict[str, Any]
+class SnippetDependencyCheckResponse(ResponseModel):
+    leaked_dependencies: list[PluginDependency]
 
 
 class SnippetUseCountResponse(ResponseModel):
@@ -58,6 +67,77 @@ class SnippetUseCountResponse(ResponseModel):
     use_count: int
 
 
+class SnippetTagResponse(ResponseModel):
+    id: str
+    name: str
+    type: str
+
+
+class SnippetAccountResponse(ResponseModel):
+    id: str
+    name: str
+    email: str
+
+
+class SnippetListItemResponse(ResponseModel):
+    id: str
+    name: str
+    description: str | None
+    type: SnippetType
+    version: int
+    use_count: int
+    is_published: bool
+    icon_info: dict[str, Any] | None
+    tags: list[SnippetTagResponse]
+    created_by: str | None
+    author_name: str | None
+    created_at: int
+    updated_by: str | None
+    updated_at: int
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int:
+        timestamp = to_timestamp(value)
+        if timestamp is None:
+            raise ValueError("timestamp is required")
+        return timestamp
+
+
+class SnippetResponse(ResponseModel):
+    id: str
+    name: str
+    description: str | None
+    type: SnippetType
+    version: int
+    use_count: int
+    is_published: bool
+    icon_info: dict[str, Any] | None
+    graph: dict[str, Any] = PydanticField(validation_alias="graph_dict")
+    input_fields: list[dict[str, Any]] = PydanticField(validation_alias="input_fields_list")
+    tags: list[SnippetTagResponse]
+    created_by: SnippetAccountResponse | None = PydanticField(validation_alias="created_by_account")
+    created_at: int
+    updated_by: SnippetAccountResponse | None = PydanticField(validation_alias="updated_by_account")
+    updated_at: int
+
+    @field_validator("created_at", "updated_at", mode="before")
+    @classmethod
+    def _normalize_timestamp(cls, value: datetime | int | None) -> int:
+        timestamp = to_timestamp(value)
+        if timestamp is None:
+            raise ValueError("timestamp is required")
+        return timestamp
+
+
+class SnippetPaginationResponse(ResponseModel):
+    data: list[SnippetListItemResponse]
+    page: int
+    limit: int
+    total: int
+    has_more: bool
+
+
 def _snippet_service() -> SnippetService:
     return SnippetService(sessionmaker(bind=db.engine, expire_on_commit=False))
 
@@ -73,11 +153,19 @@ def _normalize_snippet_list_query_args(query_args: MultiDict[str, str]) -> dict[
             indexed_tag_ids.extend((int(match.group(1)), value) for value in query_args.getlist(key))
             continue
 
-        match = _CREATOR_IDS_BRACKET_PATTERN.fullmatch(key)
+        match = _CREATOR_IDS_BRACKET_PATTERN.fullmatch(key) or _CREATORS_BRACKET_PATTERN.fullmatch(key)
         if match:
             indexed_creator_ids.extend((int(match.group(1)), value) for value in query_args.getlist(key))
             continue
 
+        if key in {"tag_ids", "creators", "creator_ids"}:
+            values = query_args.getlist(key)
+            if values:
+                normalized["creators" if key in {"creators", "creator_ids"} else key] = (
+                    values if len(values) > 1 else values[0]
+                )
+            continue
+
         value = query_args.get(key)
         if value is not None:
             normalized[key] = value
@@ -105,19 +193,17 @@ register_response_schema_models(
     SnippetImportResponse,
     SnippetDependencyCheckResponse,
     SnippetUseCountResponse,
+    SnippetListItemResponse,
+    SnippetResponse,
+    SnippetPaginationResponse,
 )
 
-# Create namespace models for marshaling
-snippet_model = console_ns.model("Snippet", snippet_fields)
-snippet_list_model = console_ns.model("SnippetList", snippet_list_fields)
-snippet_pagination_model = console_ns.model("SnippetPagination", snippet_pagination_fields)
-
 
 @console_ns.route("/workspaces/current/customized-snippets")
 class CustomizedSnippetsApi(Resource):
     @console_ns.doc("list_customized_snippets")
     @console_ns.doc(params=query_params_from_model(SnippetListQuery))
-    @console_ns.response(200, "Snippets retrieved successfully", snippet_pagination_model)
+    @console_ns.response(200, "Snippets retrieved successfully", console_ns.models[SnippetPaginationResponse.__name__])
     @setup_required
     @login_required
     @account_initialization_required
@@ -148,7 +234,7 @@ class CustomizedSnippetsApi(Resource):
 
     @console_ns.doc("create_customized_snippet")
     @console_ns.expect(console_ns.models.get(CreateSnippetPayload.__name__))
-    @console_ns.response(201, "Snippet created successfully", snippet_model)
+    @console_ns.response(201, "Snippet created successfully", console_ns.models[SnippetResponse.__name__])
     @console_ns.response(400, "Invalid request")
     @setup_required
     @login_required
@@ -191,7 +277,7 @@ class CustomizedSnippetsApi(Resource):
 @console_ns.route("/workspaces/current/customized-snippets/<uuid:snippet_id>")
 class CustomizedSnippetDetailApi(Resource):
     @console_ns.doc("get_customized_snippet")
-    @console_ns.response(200, "Snippet retrieved successfully", snippet_model)
+    @console_ns.response(200, "Snippet retrieved successfully", console_ns.models[SnippetResponse.__name__])
     @console_ns.response(404, "Snippet not found")
     @setup_required
     @login_required
@@ -212,7 +298,7 @@ class CustomizedSnippetDetailApi(Resource):
 
     @console_ns.doc("update_customized_snippet")
     @console_ns.expect(console_ns.models.get(UpdateSnippetPayload.__name__))
-    @console_ns.response(200, "Snippet updated successfully", snippet_model)
+    @console_ns.response(200, "Snippet updated successfully", console_ns.models[SnippetResponse.__name__])
     @console_ns.response(400, "Invalid request")
     @console_ns.response(404, "Snippet not found")
     @setup_required
@@ -455,9 +541,6 @@ class CustomizedSnippetUseCountIncrementApi(Resource):
     @login_required
     @account_initialization_required
     @edit_permission_required
-    @rbac_permission_required(
-        RBACResourceScope.WORKSPACE, RBACPermission.SNIPPETS_CREATE_AND_MODIFY, resource_required=False
-    )
     @with_current_tenant_id
     def post(self, current_tenant_id: str, snippet_id: str):
         """Increment snippet use count when it is inserted into a workflow."""

api/controllers/console/workspace/trigger_providers.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Any
+from typing import Any, Literal
 
 from flask import make_response, redirect, request
 from flask_restx import Resource
@@ -11,11 +11,19 @@ from configs import dify_config
 from controllers.common.errors import NotFoundError
 from controllers.common.fields import BinaryFileResponse, RedirectResponse, SimpleResultResponse
 from controllers.common.schema import register_response_schema_models, register_schema_models
+from core.entities.parameter_entities import AppSelectorScope, ModelSelectorScope, ToolSelectorScope
 from core.plugin.entities.plugin_daemon import CredentialType
 from core.plugin.impl.oauth import OAuthHandler
-from core.trigger.entities.entities import SubscriptionBuilderUpdater
+from core.tools.entities.common_entities import I18nObject
+from core.trigger.entities.api_entities import (
+    SubscriptionBuilderApiEntity,
+    TriggerProviderApiEntity,
+    TriggerProviderSubscriptionApiEntity,
+)
+from core.trigger.entities.entities import RequestLog, SubscriptionBuilderUpdater
 from core.trigger.trigger_manager import TriggerManager
 from extensions.ext_database import db
+from fields.base import ResponseModel
 from graphon.model_runtime.utils.encoders import jsonable_encoder
 from libs.login import login_required
 from models.account import Account
@@ -70,14 +78,41 @@ class TriggerOAuthClientPayload(BaseModel):
 class TriggerOAuthAuthorizeResponse(BaseModel):
     authorization_url: str
     subscription_builder_id: str
-    subscription_builder: Any
+    subscription_builder: SubscriptionBuilderApiEntity
+
+
+class TriggerProviderConfigOptionResponse(BaseModel):
+    value: str = Field(..., description="The value of the option")
+    label: I18nObject = Field(..., description="The label of the option")
+
+
+class TriggerProviderConfigResponse(BaseModel):
+    type: Literal[
+        "secret-input",
+        "text-input",
+        "select",
+        "boolean",
+        "app-selector",
+        "model-selector",
+        "array[tools]",
+    ] = Field(..., description="The type of the credentials")
+    name: str = Field(..., description="The name of the credentials")
+    scope: AppSelectorScope | ModelSelectorScope | ToolSelectorScope | None = None
+    required: bool = False
+    default: int | str | float | bool | None = None
+    options: list[TriggerProviderConfigOptionResponse] | None = None
+    multiple: bool = False
+    label: I18nObject | None = None
+    help: I18nObject | None = None
+    url: str | None = None
+    placeholder: I18nObject | None = None
 
 
 class TriggerOAuthClientResponse(BaseModel):
     configured: bool
     system_configured: bool
     custom_configured: bool
-    oauth_client_schema: Any
+    oauth_client_schema: list[TriggerProviderConfigResponse]
     custom_enabled: bool
     redirect_uri: str
     params: dict[str, Any]
@@ -87,6 +122,26 @@ class TriggerProviderOpaqueResponse(RootModel[Any]):
     root: Any
 
 
+class TriggerProviderListResponse(RootModel[list[TriggerProviderApiEntity]]):
+    root: list[TriggerProviderApiEntity]
+
+
+class TriggerSubscriptionListResponse(RootModel[list[TriggerProviderSubscriptionApiEntity]]):
+    root: list[TriggerProviderSubscriptionApiEntity]
+
+
+class TriggerSubscriptionBuilderCreateResponse(ResponseModel):
+    subscription_builder: SubscriptionBuilderApiEntity
+
+
+class TriggerSubscriptionBuilderVerifyResponse(ResponseModel):
+    verified: bool
+
+
+class TriggerSubscriptionBuilderLogsResponse(ResponseModel):
+    logs: list[RequestLog]
+
+
 register_schema_models(
     console_ns,
     TriggerSubscriptionBuilderCreatePayload,
@@ -102,6 +157,15 @@ register_response_schema_models(
     TriggerOAuthAuthorizeResponse,
     TriggerOAuthClientResponse,
     TriggerProviderOpaqueResponse,
+    TriggerProviderApiEntity,
+    TriggerProviderListResponse,
+    TriggerProviderSubscriptionApiEntity,
+    TriggerSubscriptionListResponse,
+    SubscriptionBuilderApiEntity,
+    TriggerSubscriptionBuilderCreateResponse,
+    TriggerSubscriptionBuilderVerifyResponse,
+    RequestLog,
+    TriggerSubscriptionBuilderLogsResponse,
 )
 
 
@@ -118,7 +182,7 @@ class TriggerProviderIconApi(Resource):
 
 @console_ns.route("/workspaces/current/triggers")
 class TriggerProviderListApi(Resource):
-    @console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[TriggerProviderListResponse.__name__])
     @setup_required
     @login_required
     @account_initialization_required
@@ -130,7 +194,7 @@ class TriggerProviderListApi(Resource):
 
 @console_ns.route("/workspaces/current/trigger-provider/<path:provider>/info")
 class TriggerProviderInfoApi(Resource):
-    @console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[TriggerProviderApiEntity.__name__])
     @setup_required
     @login_required
     @account_initialization_required
@@ -142,7 +206,7 @@ class TriggerProviderInfoApi(Resource):
 
 @console_ns.route("/workspaces/current/trigger-provider/<path:provider>/subscriptions/list")
 class TriggerSubscriptionListApi(Resource):
-    @console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionListResponse.__name__])
     @setup_required
     @login_required
     @edit_permission_required
@@ -172,7 +236,7 @@ class TriggerSubscriptionListApi(Resource):
 )
 class TriggerSubscriptionBuilderCreateApi(Resource):
     @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderCreatePayload.__name__])
-    @console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionBuilderCreateResponse.__name__])
     @setup_required
     @login_required
     @edit_permission_required
@@ -202,7 +266,7 @@ class TriggerSubscriptionBuilderCreateApi(Resource):
     "/workspaces/current/trigger-provider/<path:provider>/subscriptions/builder/<path:subscription_builder_id>",
 )
 class TriggerSubscriptionBuilderGetApi(Resource):
-    @console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[SubscriptionBuilderApiEntity.__name__])
     @setup_required
     @login_required
     @edit_permission_required
@@ -220,7 +284,7 @@ class TriggerSubscriptionBuilderGetApi(Resource):
 )
 class TriggerSubscriptionBuilderVerifyApi(Resource):
     @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderVerifyPayload.__name__])
-    @console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionBuilderVerifyResponse.__name__])
     @setup_required
     @login_required
     @edit_permission_required
@@ -253,7 +317,7 @@ class TriggerSubscriptionBuilderVerifyApi(Resource):
 )
 class TriggerSubscriptionBuilderUpdateApi(Resource):
     @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderUpdatePayload.__name__])
-    @console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[SubscriptionBuilderApiEntity.__name__])
     @setup_required
     @login_required
     @edit_permission_required
@@ -286,7 +350,7 @@ class TriggerSubscriptionBuilderUpdateApi(Resource):
     "/workspaces/current/trigger-provider/<path:provider>/subscriptions/builder/logs/<path:subscription_builder_id>",
 )
 class TriggerSubscriptionBuilderLogsApi(Resource):
-    @console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionBuilderLogsResponse.__name__])
     @setup_required
     @login_required
     @edit_permission_required
@@ -689,7 +753,7 @@ class TriggerOAuthClientManageApi(Resource):
 )
 class TriggerSubscriptionVerifyApi(Resource):
     @console_ns.expect(console_ns.models[TriggerSubscriptionBuilderVerifyPayload.__name__])
-    @console_ns.response(200, "Success", console_ns.models[TriggerProviderOpaqueResponse.__name__])
+    @console_ns.response(200, "Success", console_ns.models[TriggerSubscriptionBuilderVerifyResponse.__name__])
     @setup_required
     @login_required
     @edit_permission_required

api/controllers/files/__init__.py

@@ -14,11 +14,12 @@ api = ExternalApi(
 
 files_ns = Namespace("files", description="File operations", path="/")
 
-from . import image_preview, tool_files, upload
+from . import agent_drive_archive, image_preview, tool_files, upload
 
 api.add_namespace(files_ns)
 
 __all__ = [
+    "agent_drive_archive",
     "api",
     "bp",
     "files_ns",

api/controllers/files/agent_drive_archive.py

@@ -0,0 +1,67 @@
+from urllib.parse import quote
+
+from flask import Response, request
+from flask_restx import Resource
+from pydantic import BaseModel, Field
+from werkzeug.exceptions import Forbidden, NotFound
+
+from controllers.common.file_response import enforce_download_for_html
+from controllers.common.schema import register_schema_models
+from controllers.files import files_ns
+from models.agent import AgentDriveFileKind
+from services.agent_drive_service import AgentDriveError, AgentDriveService
+
+
+class AgentDriveArchiveMemberQuery(BaseModel):
+    tenant_id: str = Field(..., description="Tenant ID")
+    agent_id: str = Field(..., description="Agent ID")
+    key: str = Field(..., description="Virtual drive key")
+    archive_file_kind: AgentDriveFileKind = Field(..., description="Archive file kind")
+    archive_file_id: str = Field(..., description="Archive file id")
+    member_path: str = Field(..., description="Zip member path")
+    timestamp: str = Field(..., description="Unix timestamp")
+    nonce: str = Field(..., description="Random nonce")
+    sign: str = Field(..., description="HMAC signature")
+    as_attachment: bool = Field(default=False, description="Download as attachment")
+
+
+register_schema_models(files_ns, AgentDriveArchiveMemberQuery)
+
+
+@files_ns.route("/agent-drive/archive-member")
+class AgentDriveArchiveMemberApi(Resource):
+    @files_ns.doc("get_agent_drive_archive_member")
+    @files_ns.doc(description="Download a lazily resolved Agent Skill archive member by signed parameters")
+    def get(self):
+        args = AgentDriveArchiveMemberQuery.model_validate(request.args.to_dict(flat=True))
+        if not AgentDriveService.verify_archive_member_signature(
+            tenant_id=args.tenant_id,
+            agent_id=args.agent_id,
+            key=args.key,
+            archive_file_kind=args.archive_file_kind,
+            archive_file_id=args.archive_file_id,
+            member_path=args.member_path,
+            timestamp=args.timestamp,
+            nonce=args.nonce,
+            sign=args.sign,
+        ):
+            raise Forbidden("Invalid request.")
+        try:
+            payload, mime_type, filename = AgentDriveService().load_archive_member_for_signed_request(
+                tenant_id=args.tenant_id,
+                agent_id=args.agent_id,
+                key=args.key,
+                archive_file_kind=args.archive_file_kind,
+                archive_file_id=args.archive_file_id,
+                member_path=args.member_path,
+            )
+        except AgentDriveError as exc:
+            raise NotFound(exc.message) from exc
+
+        response = Response(payload, mimetype=mime_type, direct_passthrough=True, headers={})
+        response.headers["Content-Length"] = str(len(payload))
+        if args.as_attachment and filename:
+            encoded_filename = quote(filename)
+            response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}"
+        enforce_download_for_html(response, mime_type=mime_type, filename=filename, extension="")
+        return response

api/controllers/inner_api/__init__.py

@@ -17,8 +17,10 @@ inner_api_ns = Namespace("inner_api", description="Internal API operations", pat
 
 from . import mail as _mail
 from . import runtime_credentials as _runtime_credentials
+from .agent import tools as _agent_tools
 from .app import dsl as _app_dsl
 from .knowledge import retrieval as _knowledge_retrieval
+from .plugin import agent_config as _agent_config
 from .plugin import agent_drive as _agent_drive
 from .plugin import plugin as _plugin
 from .workspace import workspace as _workspace
@@ -26,7 +28,9 @@ from .workspace import workspace as _workspace
 api.add_namespace(inner_api_ns)
 
 __all__ = [
+    "_agent_config",
     "_agent_drive",
+    "_agent_tools",
     "_app_dsl",
     "_knowledge_retrieval",
     "_mail",

api/controllers/inner_api/agent/__init__.py

@@ -0,0 +1 @@
+"""Agent backend inner API controllers."""

api/controllers/inner_api/agent/tools.py

@@ -0,0 +1,68 @@
+"""Inner API endpoint for Agent core tool invocation."""
+
+from flask_restx import Resource
+from pydantic import ValidationError
+
+from controllers.common.schema import register_response_schema_models, register_schema_models
+from controllers.inner_api import inner_api_ns
+from controllers.inner_api.wraps import agent_inner_api_only
+from extensions.ext_database import db
+from libs.exception import BaseHTTPException
+from services.agent_tool_inner_service import AgentToolInnerService
+from services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponse
+from services.errors.agent_tool_inner import AgentToolInnerServiceError
+
+
+class AgentToolInvokeHttpError(BaseHTTPException):
+    error_code = "agent_tool_invoke_failed"
+    description = "Agent tool invocation failed."
+    code = 500
+
+    def __init__(
+        self,
+        *,
+        error_code: str | None = None,
+        description: str | None = None,
+        status_code: int | None = None,
+    ) -> None:
+        if error_code is not None:
+            self.error_code = error_code
+        if description is not None:
+            self.description = description
+        if status_code is not None:
+            self.code = status_code
+        super().__init__(self.description)
+
+
+register_schema_models(inner_api_ns, AgentToolInvokeRequest)
+register_response_schema_models(inner_api_ns, AgentToolInvokeResponse)
+
+
+@inner_api_ns.route("/agent/tools/invoke")
+class AgentToolInvokeApi(Resource):
+    """Invoke one Agent tool through the API-owned core tool runtime path."""
+
+    @agent_inner_api_only
+    @inner_api_ns.doc("inner_agent_tool_invoke")
+    @inner_api_ns.expect(inner_api_ns.models[AgentToolInvokeRequest.__name__])
+    @inner_api_ns.response(200, "Tool invoked successfully", inner_api_ns.models[AgentToolInvokeResponse.__name__])
+    def post(self) -> dict[str, object]:
+        try:
+            payload = AgentToolInvokeRequest.model_validate(inner_api_ns.payload or {})
+        except ValidationError as exc:
+            raise AgentToolInvokeHttpError(
+                error_code="invalid_request",
+                description=str(exc),
+                status_code=400,
+            ) from exc
+
+        try:
+            response = AgentToolInnerService().invoke(payload, session=db.session())
+        except AgentToolInnerServiceError as exc:
+            raise AgentToolInvokeHttpError(
+                error_code=exc.error_code,
+                description=exc.description,
+                status_code=exc.status_code,
+            ) from exc
+
+        return response.model_dump(mode="json")

api/controllers/inner_api/plugin/agent_config.py

@@ -0,0 +1,243 @@
+"""Inner API for Agent Soul-backed config assets.
+
+These endpoints are called by the dify-agent server with the inner API key.
+They resolve the requested Agent config version directly from Agent Soul JSON
+and never expose signed download URLs or drive-owned metadata.
+"""
+
+from __future__ import annotations
+
+import io
+
+from flask import request, send_file
+from flask_restx import Resource
+from pydantic import BaseModel, ValidationError
+
+from controllers.console.wraps import setup_required
+from controllers.inner_api import inner_api_ns
+from controllers.inner_api.wraps import plugin_inner_api_only
+from services.agent_config_service import (
+    AgentConfigService,
+    AgentConfigServiceError,
+    AgentConfigVersionKind,
+    ConfigPushPayload,
+)
+
+
+class _ConfigTargetQuery(BaseModel):
+    tenant_id: str
+    user_id: str | None = None
+    config_version_id: str
+    config_version_kind: AgentConfigVersionKind
+
+
+class _ConfigMutationRequest(BaseModel):
+    tenant_id: str
+    user_id: str
+    config_version_id: str
+    config_version_kind: AgentConfigVersionKind
+
+
+class _ConfigPushRequest(_ConfigMutationRequest):
+    files: list[dict] = []
+    skills: list[dict] = []
+    env_text: str | None = None
+    note: str | None = None
+
+    def to_payload(self) -> ConfigPushPayload:
+        return ConfigPushPayload.model_validate(
+            {
+                "files": self.files,
+                "skills": self.skills,
+                "env_text": self.env_text,
+                "note": self.note,
+            }
+        )
+
+
+class _ConfigEnvUpdateRequest(_ConfigMutationRequest):
+    env_text: str
+
+
+class _ConfigNoteUpdateRequest(_ConfigMutationRequest):
+    note: str
+
+
+def _target_query_from_request() -> _ConfigTargetQuery:
+    return _ConfigTargetQuery.model_validate(
+        {
+            "tenant_id": request.args.get("tenant_id"),
+            "user_id": request.args.get("user_id"),
+            "config_version_id": request.args.get("config_version_id"),
+            "config_version_kind": request.args.get("config_version_kind"),
+        }
+    )
+
+
+def _error_response(exc: AgentConfigServiceError) -> tuple[dict[str, str], int]:
+    return {"code": exc.code, "message": exc.message}, exc.status_code
+
+
+@inner_api_ns.route("/agent-config/<string:agent_id>/manifest")
+class AgentConfigManifestApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @inner_api_ns.doc("agent_config_manifest")
+    def get(self, agent_id: str):
+        try:
+            query = _target_query_from_request()
+            return AgentConfigService().manifest(
+                tenant_id=query.tenant_id,
+                agent_id=agent_id,
+                user_id=query.user_id,
+                config_version_id=query.config_version_id,
+                config_version_kind=query.config_version_kind,
+            )
+        except ValidationError as exc:
+            return {"code": "invalid_request", "message": str(exc)}, 400
+        except AgentConfigServiceError as exc:
+            return _error_response(exc)
+
+
+@inner_api_ns.route("/agent-config/<string:agent_id>/skills/<string:name>/pull")
+class AgentConfigSkillPullApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @inner_api_ns.doc("agent_config_skill_pull")
+    def get(self, agent_id: str, name: str):
+        try:
+            query = _target_query_from_request()
+            result = AgentConfigService().pull_skill(
+                tenant_id=query.tenant_id,
+                agent_id=agent_id,
+                user_id=query.user_id,
+                config_version_id=query.config_version_id,
+                config_version_kind=query.config_version_kind,
+                name=name,
+            )
+            return send_file(
+                io.BytesIO(result.payload),
+                mimetype=result.mime_type,
+                as_attachment=True,
+                download_name=result.filename,
+            )
+        except ValidationError as exc:
+            return {"code": "invalid_request", "message": str(exc)}, 400
+        except AgentConfigServiceError as exc:
+            return _error_response(exc)
+
+
+@inner_api_ns.route("/agent-config/<string:agent_id>/skills/<string:name>/inspect")
+class AgentConfigSkillInspectApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @inner_api_ns.doc("agent_config_skill_inspect")
+    def get(self, agent_id: str, name: str):
+        try:
+            query = _target_query_from_request()
+            return AgentConfigService().inspect_skill(
+                tenant_id=query.tenant_id,
+                agent_id=agent_id,
+                user_id=query.user_id,
+                config_version_id=query.config_version_id,
+                config_version_kind=query.config_version_kind,
+                name=name,
+            )
+        except ValidationError as exc:
+            return {"code": "invalid_request", "message": str(exc)}, 400
+        except AgentConfigServiceError as exc:
+            return _error_response(exc)
+
+
+@inner_api_ns.route("/agent-config/<string:agent_id>/files/<string:name>/pull")
+class AgentConfigFilePullApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @inner_api_ns.doc("agent_config_file_pull")
+    def get(self, agent_id: str, name: str):
+        try:
+            query = _target_query_from_request()
+            result = AgentConfigService().pull_file(
+                tenant_id=query.tenant_id,
+                agent_id=agent_id,
+                user_id=query.user_id,
+                config_version_id=query.config_version_id,
+                config_version_kind=query.config_version_kind,
+                name=name,
+            )
+            return send_file(
+                io.BytesIO(result.payload),
+                mimetype=result.mime_type,
+                as_attachment=True,
+                download_name=result.filename,
+            )
+        except ValidationError as exc:
+            return {"code": "invalid_request", "message": str(exc)}, 400
+        except AgentConfigServiceError as exc:
+            return _error_response(exc)
+
+
+@inner_api_ns.route("/agent-config/<string:agent_id>/push")
+class AgentConfigPushApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @inner_api_ns.doc("agent_config_push")
+    def post(self, agent_id: str):
+        try:
+            body = _ConfigPushRequest.model_validate(request.get_json(silent=True) or {})
+            return AgentConfigService().push(
+                tenant_id=body.tenant_id,
+                agent_id=agent_id,
+                user_id=body.user_id,
+                config_version_id=body.config_version_id,
+                config_version_kind=body.config_version_kind,
+                payload=body.to_payload(),
+            )
+        except ValidationError as exc:
+            return {"code": "invalid_request", "message": str(exc)}, 400
+        except AgentConfigServiceError as exc:
+            return _error_response(exc)
+
+
+@inner_api_ns.route("/agent-config/<string:agent_id>/env")
+class AgentConfigEnvApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @inner_api_ns.doc("agent_config_env")
+    def patch(self, agent_id: str):
+        try:
+            body = _ConfigEnvUpdateRequest.model_validate(request.get_json(silent=True) or {})
+            return AgentConfigService().update_env(
+                tenant_id=body.tenant_id,
+                agent_id=agent_id,
+                user_id=body.user_id,
+                config_version_id=body.config_version_id,
+                config_version_kind=body.config_version_kind,
+                env_text=body.env_text,
+            )
+        except ValidationError as exc:
+            return {"code": "invalid_request", "message": str(exc)}, 400
+        except AgentConfigServiceError as exc:
+            return _error_response(exc)
+
+
+@inner_api_ns.route("/agent-config/<string:agent_id>/note")
+class AgentConfigNoteApi(Resource):
+    @setup_required
+    @plugin_inner_api_only
+    @inner_api_ns.doc("agent_config_note")
+    def put(self, agent_id: str):
+        try:
+            body = _ConfigNoteUpdateRequest.model_validate(request.get_json(silent=True) or {})
+            return AgentConfigService().update_note(
+                tenant_id=body.tenant_id,
+                agent_id=agent_id,
+                user_id=body.user_id,
+                config_version_id=body.config_version_id,
+                config_version_kind=body.config_version_kind,
+                note=body.note,
+            )
+        except ValidationError as exc:
+            return {"code": "invalid_request", "message": str(exc)}, 400
+        except AgentConfigServiceError as exc:
+            return _error_response(exc)

api/controllers/inner_api/wraps.py

@@ -95,3 +95,15 @@ def plugin_inner_api_only[**P, R](view: Callable[P, R]) -> Callable[P, R]:
         return view(*args, **kwargs)
 
     return decorated
+
+
+def agent_inner_api_only[**P, R](view: Callable[P, R]) -> Callable[P, R]:
+    """Temporary alias for agent-backend inner API callers.
+
+    Agent tool and knowledge calls currently share the same trusted
+    `dify-agent -> Dify API` transport credentials as the plugin inner bridge.
+    Keep the wrapper name agent-specific so the controller surface does not grow
+    more plugin-only semantics while auth settings stay shared.
+    """
+
+    return plugin_inner_api_only(view)

api/controllers/openapi/_errors.py

@@ -34,7 +34,6 @@ class OpenApiErrorCode(StrEnum):
     # transport-generic (resolved from HTTP status for plain werkzeug raises)
     BAD_REQUEST = "bad_request"
     UNAUTHORIZED = "unauthorized"
-    TOKEN_EXPIRED = "token_expired"
     FORBIDDEN = "forbidden"
     NOT_FOUND = "not_found"
     METHOD_NOT_ALLOWED = "method_not_allowed"
@@ -224,19 +223,6 @@ class OpenApiErrorFormatter:
         return isinstance(part, (str, int)) and not isinstance(part, bool)
 
 
-class InvalidBearer(OpenApiError):  # noqa: N818
-    code = 401
-    error_code = OpenApiErrorCode.UNAUTHORIZED
-    description = "Invalid or unknown bearer token."
-
-
-class SessionExpired(OpenApiError):  # noqa: N818
-    code = 401
-    error_code = OpenApiErrorCode.TOKEN_EXPIRED
-    description = "Your session has expired."
-    hint = "Re-authenticate to continue (e.g. re-run your login command)."
-
-
 class FilenameNotExists(OpenApiError):  # noqa: N818
     code = 400
     error_code = OpenApiErrorCode.FILENAME_NOT_EXISTS

api/controllers/openapi/auth/pipeline.py

@@ -17,7 +17,6 @@ from flask_login import user_logged_in
 from werkzeug.exceptions import Forbidden, NotFound, Unauthorized
 
 from controllers.openapi._audit import emit_wrong_surface
-from controllers.openapi._errors import InvalidBearer, SessionExpired
 from controllers.openapi.auth.data import (
     AuthData,
     Edition,
@@ -29,9 +28,7 @@ from controllers.openapi.auth.data import (
 from controllers.openapi.auth.flow import When
 from libs.oauth_bearer import (
     AuthContext,
-    InvalidBearerError,
     Scope,
-    TokenExpiredError,
     TokenType,
     extract_bearer,
     get_authenticator,
@@ -220,12 +217,7 @@ class PipelineRouter:
         if not token:
             raise Unauthorized("bearer required")
 
-        try:
-            identity = get_authenticator().authenticate(token)
-        except TokenExpiredError:
-            raise SessionExpired()
-        except InvalidBearerError:
-            raise InvalidBearer()
+        identity = get_authenticator().authenticate(token)
 
         if allowed_token_types is not None and identity.token_type not in allowed_token_types:
             emit_wrong_surface(

api/controllers/service_api/app/annotation.py

@@ -21,6 +21,7 @@ from services.annotation_service import (
     InsertAnnotationArgs,
     UpdateAnnotationArgs,
 )
+from services.app_ref_service import AppRefService
 
 
 class AnnotationCreatePayload(BaseModel):
@@ -282,9 +283,9 @@ class AnnotationUpdateDeleteApi(Resource):
         """Update an existing annotation."""
         payload = AnnotationCreatePayload.model_validate(service_api_ns.payload or {})
         update_args: UpdateAnnotationArgs = {"question": payload.question, "answer": payload.answer}
-        annotation = AppAnnotationService.update_app_annotation_directly(
-            update_args, app_model.id, str(annotation_id), db.session
-        )
+        app_ref = AppRefService.create_app_ref(app_model)
+        annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
+        annotation = AppAnnotationService.update_app_annotation_directly(update_args, annotation_ref, db.session)
         response = Annotation.model_validate(annotation, from_attributes=True)
         return response.model_dump(mode="json")
 
@@ -313,5 +314,7 @@ class AnnotationUpdateDeleteApi(Resource):
     @edit_permission_required
     def delete(self, app_model: App, annotation_id: UUID):
         """Delete an annotation."""
-        AppAnnotationService.delete_app_annotation(app_model.id, str(annotation_id), db.session)
+        app_ref = AppRefService.create_app_ref(app_model)
+        annotation_ref = AppRefService.create_annotation_ref(app_ref, str(annotation_id))
+        AppAnnotationService.delete_app_annotation(annotation_ref, db.session)
         return "", 204

api/controllers/service_api/app/audio.py

@@ -26,6 +26,7 @@ from core.errors.error import ModelCurrentlyNotSupportError, ProviderTokenNotIni
 from extensions.ext_database import db
 from graphon.model_runtime.errors.invoke import InvokeError
 from models.model import App, EndUser
+from services.app_ref_service import AppRefService
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,
@@ -177,13 +178,21 @@ class TextApi(Resource):
             message_id = payload.message_id
             text = payload.text
             voice = payload.voice
+            message_ref = None
+            if message_id:
+                app_ref = AppRefService.create_app_ref(app_model)
+                message_ref = AppRefService.create_message_ref(
+                    app_ref,
+                    message_id,
+                    end_user_id=end_user.id,
+                )
             response = AudioService.transcript_tts(
                 app_model=app_model,
                 session=db.session,
                 text=text,
                 voice=voice,
                 end_user=end_user.external_user_id,
-                message_id=message_id,
+                message_ref=message_ref,
             )
 
             return response

api/controllers/service_api/dataset/dataset.py

@@ -519,6 +519,7 @@ class DatasetListApi(DatasetApiResource):
                 embedding_model_name=payload.embedding_model,
                 retrieval_model=payload.retrieval_model,
                 summary_index_setting=payload.summary_index_setting,
+                session=db.session,
             )
         except services.errors.dataset.DatasetNameDuplicateError:
             raise DatasetNameDuplicateError()
@@ -561,11 +562,11 @@ class DatasetApi(DatasetApiResource):
     )
     def get(self, _, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
         data = _dump_service_dataset_detail(dataset)
@@ -597,7 +598,7 @@ class DatasetApi(DatasetApiResource):
                 retrieval_model_dict["search_method"] = "keyword_search"
 
         if data.get("permission") == "partial_members":
-            part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+            part_users_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
             data.update({"partial_member_list": part_users_list})
 
         return _dump_service_dataset_with_partial_members(data), 200
@@ -635,7 +636,7 @@ class DatasetApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def patch(self, _, dataset_id: UUID):
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
 
@@ -676,9 +677,10 @@ class DatasetApi(DatasetApiResource):
                 dataset,
                 str(payload.permission) if payload.permission else None,
                 payload.partial_member_list,
+                db.session,
             )
 
-        dataset = DatasetService.update_dataset(dataset_id_str, update_data, current_user)
+        dataset = DatasetService.update_dataset(dataset_id_str, update_data, current_user, db.session)
 
         if dataset is None:
             raise NotFound("Dataset not found.")
@@ -688,12 +690,14 @@ class DatasetApi(DatasetApiResource):
         tenant_id = current_user.current_tenant_id
 
         if payload.partial_member_list and payload.permission == DatasetPermissionEnum.PARTIAL_TEAM:
-            DatasetPermissionService.update_partial_member_list(tenant_id, dataset_id_str, payload.partial_member_list)
+            DatasetPermissionService.update_partial_member_list(
+                tenant_id, dataset_id_str, payload.partial_member_list, db.session
+            )
         # clear partial member list when permission is only_me or all_team_members
         elif payload.permission in {DatasetPermissionEnum.ONLY_ME, DatasetPermissionEnum.ALL_TEAM}:
-            DatasetPermissionService.clear_partial_member_list(dataset_id_str)
+            DatasetPermissionService.clear_partial_member_list(dataset_id_str, db.session)
 
-        partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str)
+        partial_member_list = DatasetPermissionService.get_dataset_partial_member_list(dataset_id_str, db.session)
         result_data.update({"partial_member_list": partial_member_list})
 
         return _dump_service_dataset_with_partial_members(result_data), 200
@@ -746,8 +750,8 @@ class DatasetApi(DatasetApiResource):
         dataset_id_str = str(dataset_id)
 
         try:
-            if DatasetService.delete_dataset(dataset_id_str, current_user):
-                DatasetPermissionService.clear_partial_member_list(dataset_id_str)
+            if DatasetService.delete_dataset(dataset_id_str, current_user, db.session):
+                DatasetPermissionService.clear_partial_member_list(dataset_id_str, db.session)
                 return "", 204
             else:
                 raise NotFound("Dataset not found.")
@@ -812,14 +816,14 @@ class DocumentStatusApi(DatasetApiResource):
             InvalidActionError: If the action is invalid or cannot be performed.
         """
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
 
         if dataset is None:
             raise NotFound("Dataset not found.")
 
         # Check user's permission
         try:
-            DatasetService.check_dataset_permission(dataset, current_user)
+            DatasetService.check_dataset_permission(dataset, current_user, db.session)
         except services.errors.account.NoPermissionError as e:
             raise Forbidden(str(e))
 
@@ -831,7 +835,7 @@ class DocumentStatusApi(DatasetApiResource):
         document_ids = data.get("document_ids", [])
 
         try:
-            DocumentService.batch_update_document_status(dataset, document_ids, action, current_user)
+            DocumentService.batch_update_document_status(dataset, document_ids, action, current_user, db.session)
         except services.errors.document.DocumentIndexingError as e:
             raise InvalidActionError(str(e))
         except ValueError as e:
@@ -939,9 +943,11 @@ class DatasetTagsApi(DatasetApiResource):
 
         payload = TagUpdatePayload.model_validate(service_api_ns.payload or {})
         tag_id = payload.tag_id
-        tag = TagService.update_tags(UpdateTagServicePayload(name=payload.name), tag_id, db.session)
+        tag = TagService.update_tags(
+            UpdateTagServicePayload(name=payload.name), tag_id, db.session, tag_type=TagType.KNOWLEDGE
+        )
 
-        binding_count = TagService.get_tag_binding_count(tag_id, db.session)
+        binding_count = TagService.get_tag_binding_count(tag_id, db.session, tag_type=TagType.KNOWLEDGE)
 
         response = dump_response(
             KnowledgeTagResponse,
@@ -971,7 +977,7 @@ class DatasetTagsApi(DatasetApiResource):
     def delete(self, _):
         """Delete a knowledge type tag."""
         payload = TagDeletePayload.model_validate(service_api_ns.payload or {})
-        TagService.delete_tag(payload.tag_id, db.session)
+        TagService.delete_tag(payload.tag_id, db.session, tag_type=TagType.KNOWLEDGE)
 
         return "", 204
 

api/controllers/service_api/dataset/document.py

@@ -400,6 +400,7 @@ def _create_document_by_text(tenant_id: str, dataset_id: UUID) -> tuple[Mapping[
             account=current_user,
             dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
             created_from="api",
+            session=db.session,
         )
     except ProviderTokenNotInitError as ex:
         raise ProviderNotInitializeError(ex.description)
@@ -459,6 +460,7 @@ def _update_document_by_text(tenant_id: str, dataset_id: UUID, document_id: UUID
             account=current_user,
             dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
             created_from="api",
+            session=db.session,
         )
     except ProviderTokenNotInitError as ex:
         raise ProviderNotInitializeError(ex.description)
@@ -756,6 +758,7 @@ class DocumentAddByFileApi(DatasetApiResource):
                 account=dataset.created_by_account,
                 dataset_process_rule=dataset_process_rule,
                 created_from="api",
+                session=db.session,
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -832,6 +835,7 @@ def _update_document_by_file(tenant_id: str, dataset_id: UUID, document_id: UUID
             account=dataset.created_by_account,
             dataset_process_rule=dataset.latest_process_rule if "process_rule" not in args else None,
             created_from="api",
+            session=db.session,
         )
     except ProviderTokenNotInitError as ex:
         raise ProviderNotInitializeError(ex.description)
@@ -1002,6 +1006,7 @@ class DocumentBatchDownloadZipApi(DatasetApiResource):
             document_ids=[str(document_id) for document_id in payload.document_ids],
             tenant_id=str(tenant_id),
             current_user=current_user,
+            session=db.session,
         )
 
         with ExitStack() as stack:
@@ -1058,7 +1063,7 @@ class DocumentIndexingStatusApi(DatasetApiResource):
         if not dataset:
             raise NotFound("Dataset not found.")
         # get documents
-        documents = DocumentService.get_batch_documents(dataset_id_str, batch)
+        documents = DocumentService.get_batch_documents(dataset_id_str, batch, db.session)
         if not documents:
             raise NotFound("Documents not found.")
         documents_status = []
@@ -1134,7 +1139,7 @@ class DocumentDownloadApi(DatasetApiResource):
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def get(self, tenant_id, dataset_id: UUID, document_id: UUID):
         dataset = self.get_dataset(str(dataset_id), str(tenant_id))
-        document = DocumentService.get_document(dataset.id, str(document_id))
+        document = DocumentService.get_document(dataset.id, str(document_id), session=db.session)
 
         if not document:
             raise NotFound("Document not found.")
@@ -1142,7 +1147,7 @@ class DocumentDownloadApi(DatasetApiResource):
         if document.tenant_id != str(tenant_id):
             raise Forbidden("No permission.")
 
-        return {"url": DocumentService.get_document_download_url(document)}
+        return {"url": DocumentService.get_document_download_url(document, db.session)}
 
 
 @service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
@@ -1190,7 +1195,7 @@ class DocumentApi(DatasetApiResource):
 
         dataset = self.get_dataset(dataset_id_str, tenant_id)
 
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
 
         if not document:
             raise NotFound("Document not found.")
@@ -1215,7 +1220,7 @@ class DocumentApi(DatasetApiResource):
         if metadata == "only":
             response = {"id": document.id, "doc_type": document.doc_type, "doc_metadata": document.doc_metadata_details}
         elif metadata == "without":
-            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str)
+            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session)
             document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
             data_source_info = document.data_source_detail_dict
             response = {
@@ -1250,7 +1255,7 @@ class DocumentApi(DatasetApiResource):
                 "need_summary": document.need_summary if document.need_summary is not None else False,
             }
         else:
-            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str)
+            dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, db.session)
             document_process_rules = document.dataset_process_rule.to_dict() if document.dataset_process_rule else {}
             data_source_info = document.data_source_detail_dict
             response = {
@@ -1345,7 +1350,7 @@ class DocumentApi(DatasetApiResource):
         if not dataset:
             raise ValueError("Dataset does not exist.")
 
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
 
         # 404 if document not found
         if document is None:
@@ -1357,7 +1362,7 @@ class DocumentApi(DatasetApiResource):
 
         try:
             # delete document
-            DocumentService.delete_document(document)
+            DocumentService.delete_document(document, db.session)
         except services.errors.document.DocumentIndexingError:
             raise DocumentIndexingError("Cannot delete document during indexing.")
 

api/controllers/service_api/dataset/metadata.py

@@ -81,10 +81,10 @@ class DatasetMetadataCreateServiceApi(DatasetApiResource):
         metadata_args = MetadataArgs.model_validate(service_api_ns.payload or {})
 
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         metadata = MetadataService.create_metadata(db.session(), dataset_id_str, metadata_args)
         return dump_response(DatasetMetadataResponse, metadata), 201
@@ -116,7 +116,7 @@ class DatasetMetadataCreateServiceApi(DatasetApiResource):
     def get(self, tenant_id, dataset_id: UUID):
         """Get all metadata for a dataset."""
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
         metadata = MetadataService.get_dataset_metadatas(db.session(), dataset)
@@ -154,10 +154,10 @@ class DatasetMetadataServiceApi(DatasetApiResource):
 
         dataset_id_str = str(dataset_id)
         metadata_id_str = str(metadata_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         metadata = MetadataService.update_metadata_name(db.session(), dataset_id_str, metadata_id_str, payload.name)
         return dump_response(DatasetMetadataResponse, metadata), 200
@@ -189,10 +189,10 @@ class DatasetMetadataServiceApi(DatasetApiResource):
         """Delete metadata."""
         dataset_id_str = str(dataset_id)
         metadata_id_str = str(metadata_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         MetadataService.delete_metadata(db.session(), dataset_id_str, metadata_id_str)
         return "", 204
@@ -257,10 +257,10 @@ class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource):
     def post(self, tenant_id, dataset_id: UUID, action: Literal["enable", "disable"]):
         """Enable or disable built-in metadata field."""
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         match action:
             case "enable":
@@ -303,10 +303,10 @@ class DocumentMetadataEditServiceApi(DatasetApiResource):
     def post(self, tenant_id, dataset_id: UUID):
         """Update metadata for multiple documents."""
         dataset_id_str = str(dataset_id)
-        dataset = DatasetService.get_dataset(dataset_id_str)
+        dataset = DatasetService.get_dataset(dataset_id_str, db.session)
         if dataset is None:
             raise NotFound("Dataset not found.")
-        DatasetService.check_dataset_permission(dataset, current_user)
+        DatasetService.check_dataset_permission(dataset, current_user, db.session)
 
         metadata_args = MetadataOperationData.model_validate(service_api_ns.payload or {})
 

api/controllers/service_api/dataset/segment.py

@@ -37,7 +37,8 @@ from fields.segment_fields import (
 from graphon.model_runtime.entities.model_entities import ModelType
 from libs.helper import dump_response
 from libs.login import current_account_with_tenant
-from models.dataset import Dataset, DocumentSegment
+from models.dataset import Dataset, Document, DocumentSegment
+from services.dataset_ref_service import DatasetRefService, SegmentRef
 from services.dataset_service import DatasetService, DocumentService, SegmentService
 from services.entities.knowledge_entities.knowledge_entities import SegmentUpdateArgs
 from services.errors.chunk import ChildChunkDeleteIndexError, ChildChunkIndexingError
@@ -127,6 +128,21 @@ register_response_schema_models(
 )
 
 
+def _get_segment_for_document(
+    dataset: Dataset, document: Document, segment_id: str
+) -> tuple[SegmentRef, DocumentSegment]:
+    dataset_ref = DatasetRefService.create_dataset_ref(dataset)
+    document_ref = DatasetRefService.create_document_ref(dataset_ref, document)
+    if document_ref is None:
+        raise NotFound("Document not found.")
+
+    segment_ref = DatasetRefService.create_segment_ref(document_ref, segment_id)
+    segment = SegmentService.get_segment_by_ref(segment_ref)
+    if not segment:
+        raise NotFound("Segment not found.")
+    return segment_ref, segment
+
+
 @service_api_ns.route("/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/segments")
 class SegmentApi(DatasetApiResource):
     """Resource for segments."""
@@ -175,7 +191,7 @@ class SegmentApi(DatasetApiResource):
             raise NotFound("Dataset not found.")
         document_id_str = str(document_id)
         # check document
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         if document.indexing_status != "completed":
@@ -210,7 +226,9 @@ class SegmentApi(DatasetApiResource):
 
         for args_item in segment_items:
             SegmentService.segment_create_args_validate(args_item, document)
-        segments = cast(list[DocumentSegment], SegmentService.multi_create_segment(segment_items, document, dataset))
+        segments = cast(
+            list[DocumentSegment], SegmentService.multi_create_segment(segment_items, document, dataset, db.session)
+        )
         segment_ids = [segment.id for segment in segments]
         summaries: dict[str, str | None] = {}
         if segment_ids:
@@ -267,7 +285,7 @@ class SegmentApi(DatasetApiResource):
             raise NotFound("Dataset not found.")
         document_id_str = str(document_id)
         # check document
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         # check embedding model setting
@@ -337,7 +355,7 @@ class DatasetSegmentApi(DatasetApiResource):
     )
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def delete(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
-        _, current_tenant_id = current_account_with_tenant()
+        current_account_with_tenant()
         dataset_id_str = str(dataset_id)
         # check dataset
         dataset = db.session.scalar(
@@ -349,15 +367,12 @@ class DatasetSegmentApi(DatasetApiResource):
         DatasetService.check_dataset_model_setting(dataset)
         document_id_str = str(document_id)
         # check document
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         segment_id_str = str(segment_id)
-        # check segment
-        segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
-        if not segment:
-            raise NotFound("Segment not found.")
-        SegmentService.delete_segment(segment, document, dataset)
+        _, segment = _get_segment_for_document(dataset, document, segment_id_str)
+        SegmentService.delete_segment(segment, document, dataset, db.session)
         return "", 204
 
     @service_api_ns.doc(
@@ -395,7 +410,7 @@ class DatasetSegmentApi(DatasetApiResource):
         DatasetService.check_dataset_model_setting(dataset)
         document_id_str = str(document_id)
         # check document
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
@@ -415,14 +430,11 @@ class DatasetSegmentApi(DatasetApiResource):
             except ProviderTokenNotInitError as ex:
                 raise ProviderNotInitializeError(ex.description)
         segment_id_str = str(segment_id)
-        # check segment
-        segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
-        if not segment:
-            raise NotFound("Segment not found.")
+        _, segment = _get_segment_for_document(dataset, document, segment_id_str)
 
         payload = SegmentUpdatePayload.model_validate(service_api_ns.payload or {})
 
-        updated_segment = SegmentService.update_segment(payload.segment, segment, document, dataset)
+        updated_segment = SegmentService.update_segment(payload.segment, segment, document, dataset, db.session)
         summary = SummaryIndexService.get_segment_summary(segment_id=updated_segment.id, dataset_id=dataset_id_str)
         response = {
             "data": segment_response_with_summary(updated_segment, summary.summary_content if summary else None),
@@ -457,7 +469,7 @@ class DatasetSegmentApi(DatasetApiResource):
         service_api_ns.models[SegmentDetailResponse.__name__],
     )
     def get(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
-        _, current_tenant_id = current_account_with_tenant()
+        current_account_with_tenant()
         dataset_id_str = str(dataset_id)
         # check dataset
         dataset = db.session.scalar(
@@ -469,14 +481,11 @@ class DatasetSegmentApi(DatasetApiResource):
         DatasetService.check_dataset_model_setting(dataset)
         document_id_str = str(document_id)
         # check document
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
         segment_id_str = str(segment_id)
-        # check segment
-        segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
-        if not segment:
-            raise NotFound("Segment not found.")
+        _, segment = _get_segment_for_document(dataset, document, segment_id_str)
 
         summary = SummaryIndexService.get_segment_summary(segment_id=segment.id, dataset_id=dataset_id_str)
         response = {
@@ -533,15 +542,12 @@ class ChildChunkApi(DatasetApiResource):
 
         document_id_str = str(document_id)
         # check document
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
 
         segment_id_str = str(segment_id)
-        # check segment
-        segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
-        if not segment:
-            raise NotFound("Segment not found.")
+        _, segment = _get_segment_for_document(dataset, document, segment_id_str)
 
         # check embedding model setting
         if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
@@ -564,7 +570,7 @@ class ChildChunkApi(DatasetApiResource):
         payload = ChildChunkCreatePayload.model_validate(service_api_ns.payload or {})
 
         try:
-            child_chunk = SegmentService.create_child_chunk(payload.content, segment, document, dataset)
+            child_chunk = SegmentService.create_child_chunk(payload.content, segment, document, dataset, db.session)
         except ChildChunkIndexingServiceError as e:
             raise ChildChunkIndexingError(str(e))
 
@@ -595,7 +601,7 @@ class ChildChunkApi(DatasetApiResource):
         service_api_ns.models[ChildChunkListResponse.__name__],
     )
     def get(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID):
-        _, current_tenant_id = current_account_with_tenant()
+        current_account_with_tenant()
         """Get child chunks."""
         dataset_id_str = str(dataset_id)
         # check dataset
@@ -607,15 +613,12 @@ class ChildChunkApi(DatasetApiResource):
 
         document_id_str = str(document_id)
         # check document
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
 
         segment_id_str = str(segment_id)
-        # check segment
-        segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
-        if not segment:
-            raise NotFound("Segment not found.")
+        _get_segment_for_document(dataset, document, segment_id_str)
 
         args = query_params_from_request(ChildChunkListQuery, use_defaults_for_malformed_ints=True)
 
@@ -665,7 +668,7 @@ class DatasetChildChunkApi(DatasetApiResource):
     @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def delete(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID, child_chunk_id: UUID):
-        _, current_tenant_id = current_account_with_tenant()
+        current_account_with_tenant()
         """Delete child chunk."""
         dataset_id_str = str(dataset_id)
         # check dataset
@@ -677,34 +680,21 @@ class DatasetChildChunkApi(DatasetApiResource):
 
         document_id_str = str(document_id)
         # check document
-        document = DocumentService.get_document(dataset.id, document_id_str)
+        document = DocumentService.get_document(dataset.id, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
 
         segment_id_str = str(segment_id)
-        # check segment
-        segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
-        if not segment:
-            raise NotFound("Segment not found.")
-
-        # validate segment belongs to the specified document
-        if segment.document_id != document_id_str:
-            raise NotFound("Document not found.")
+        segment_ref, _ = _get_segment_for_document(dataset, document, segment_id_str)
 
         child_chunk_id_str = str(child_chunk_id)
         # check child chunk
-        child_chunk = SegmentService.get_child_chunk_by_id(
-            child_chunk_id=child_chunk_id_str, tenant_id=current_tenant_id
-        )
+        child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref)
         if not child_chunk:
             raise NotFound("Child chunk not found.")
 
-        # validate child chunk belongs to the specified segment
-        if child_chunk.segment_id != segment.id:
-            raise NotFound("Child chunk not found.")
-
         try:
-            SegmentService.delete_child_chunk(child_chunk, dataset)
+            SegmentService.delete_child_chunk(child_chunk, dataset, db.session)
         except ChildChunkDeleteIndexServiceError as e:
             raise ChildChunkDeleteIndexError(str(e))
 
@@ -739,7 +729,7 @@ class DatasetChildChunkApi(DatasetApiResource):
     @cloud_edition_billing_knowledge_limit_check("add_segment", "dataset")
     @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
     def patch(self, tenant_id: str, dataset_id: UUID, document_id: UUID, segment_id: UUID, child_chunk_id: UUID):
-        _, current_tenant_id = current_account_with_tenant()
+        current_account_with_tenant()
         """Update child chunk."""
         dataset_id_str = str(dataset_id)
         # check dataset
@@ -751,37 +741,26 @@ class DatasetChildChunkApi(DatasetApiResource):
 
         document_id_str = str(document_id)
         # get document
-        document = DocumentService.get_document(dataset_id_str, document_id_str)
+        document = DocumentService.get_document(dataset_id_str, document_id_str, session=db.session)
         if not document:
             raise NotFound("Document not found.")
 
         segment_id_str = str(segment_id)
-        # get segment
-        segment = SegmentService.get_segment_by_id(segment_id=segment_id_str, tenant_id=current_tenant_id)
-        if not segment:
-            raise NotFound("Segment not found.")
-
-        # validate segment belongs to the specified document
-        if segment.document_id != document_id_str:
-            raise NotFound("Segment not found.")
+        segment_ref, segment = _get_segment_for_document(dataset, document, segment_id_str)
 
         child_chunk_id_str = str(child_chunk_id)
         # get child chunk
-        child_chunk = SegmentService.get_child_chunk_by_id(
-            child_chunk_id=child_chunk_id_str, tenant_id=current_tenant_id
-        )
+        child_chunk = SegmentService.get_child_chunk_by_segment_ref(child_chunk_id_str, segment_ref)
         if not child_chunk:
             raise NotFound("Child chunk not found.")
 
-        # validate child chunk belongs to the specified segment
-        if child_chunk.segment_id != segment.id:
-            raise NotFound("Child chunk not found.")
-
         # validate args
         payload = ChildChunkUpdatePayload.model_validate(service_api_ns.payload or {})
 
         try:
-            child_chunk = SegmentService.update_child_chunk(payload.content, child_chunk, segment, document, dataset)
+            child_chunk = SegmentService.update_child_chunk(
+                payload.content, child_chunk, segment, document, dataset, db.session
+            )
         except ChildChunkIndexingServiceError as e:
             raise ChildChunkIndexingError(str(e))
 

api/controllers/service_api/wraps.py

@@ -12,6 +12,7 @@ from flask_restx import Resource
 from flask_restx.utils import merge
 from pydantic import BaseModel
 from sqlalchemy import select
+from sqlalchemy.orm import sessionmaker
 from werkzeug.exceptions import Forbidden, NotFound, Unauthorized
 
 from configs import dify_config
@@ -269,8 +270,8 @@ def cloud_edition_billing_rate_limit_check[**P, R](
                             subscription_plan=knowledge_rate_limit.subscription_plan,
                             operation="knowledge",
                         )
-                        db.session.add(rate_limit_log)
-                        db.session.commit()
+                        with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session:
+                            session.add(rate_limit_log)
                         raise Forbidden(
                             "Sorry, you have reached the knowledge base request rate limit of your subscription."
                         )

api/controllers/web/audio.py

@@ -26,6 +26,7 @@ from extensions.ext_database import db
 from graphon.model_runtime.errors.invoke import InvokeError
 from libs.helper import uuid_value
 from models.model import App, EndUser
+from services.app_ref_service import AppRefService
 from services.audio_service import AudioService
 from services.errors.audio import (
     AudioTooLargeServiceError,
@@ -130,13 +131,21 @@ class TextApi(WebApiResource):
             message_id = payload.message_id
             text = payload.text
             voice = payload.voice
+            message_ref = None
+            if message_id:
+                app_ref = AppRefService.create_app_ref(app_model)
+                message_ref = AppRefService.create_message_ref(
+                    app_ref,
+                    message_id,
+                    end_user_id=end_user.id,
+                )
             response = AudioService.transcript_tts(
                 app_model=app_model,
                 session=db.session,
                 text=text,
                 voice=voice,
                 end_user=end_user.external_user_id,
-                message_id=message_id,
+                message_ref=message_ref,
             )
 
             return response

api/controllers/web/site.py

@@ -14,7 +14,7 @@ from fields.base import ResponseModel
 from libs.helper import AppIconUrlField
 from models.account import TenantStatus
 from models.model import App, EndUser, Site
-from services.feature_service import FeatureService
+from services.feature_service import FeatureModel, FeatureService
 
 
 class AppSiteModelConfigResponse(ResponseModel):
@@ -38,6 +38,7 @@ class AppSiteResponse(ResponseModel):
     description: str | None = None
     copyright: str | None = None
     privacy_policy: str | None = None
+    input_placeholder: str | None = None
     custom_disclaimer: str | None = None
     default_language: str | None = None
     prompt_public: bool | None = None
@@ -84,6 +85,7 @@ class AppSiteApi(WebApiResource):
         "description": fields.String,
         "copyright": fields.String,
         "privacy_policy": fields.String,
+        "input_placeholder": fields.String,
         "custom_disclaimer": fields.String,
         "default_language": fields.String,
         "prompt_public": fields.Boolean,
@@ -127,9 +129,15 @@ class AppSiteApi(WebApiResource):
         if app_model.tenant and app_model.tenant.status == TenantStatus.ARCHIVE:
             raise Forbidden()
 
-        can_replace_logo = FeatureService.get_features(app_model.tenant_id, exclude_vector_space=True).can_replace_logo
+        features = FeatureService.get_features(app_model.tenant_id, exclude_vector_space=True)
 
-        return AppSiteInfo(app_model.tenant, app_model, site, end_user.id, can_replace_logo)
+        return AppSiteInfo(
+            app_model.tenant,
+            app_model,
+            serialize_runtime_site(site, features),
+            end_user.id,
+            features.can_replace_logo,
+        )
 
 
 class AppSiteInfo:
@@ -164,7 +172,23 @@ def serialize_site(site: Site) -> dict[str, Any]:
     return cast(dict[str, Any], marshal(site, AppSiteApi.site_fields))
 
 
+def serialize_runtime_site(site: Site, features: FeatureModel) -> dict[str, Any]:
+    site_payload = serialize_site(site)
+    if not features.billing.enabled or features.webapp_copyright_enabled:
+        return site_payload
+
+    site_payload["copyright"] = None
+    site_payload["input_placeholder"] = None
+    return site_payload
+
+
 def serialize_app_site_payload(app_model: App, site: Site, end_user_id: str | None) -> dict[str, Any]:
-    can_replace_logo = FeatureService.get_features(app_model.tenant_id, exclude_vector_space=True).can_replace_logo
-    app_site_info = AppSiteInfo(app_model.tenant, app_model, site, end_user_id, can_replace_logo)
+    features = FeatureService.get_features(app_model.tenant_id, exclude_vector_space=True)
+    app_site_info = AppSiteInfo(
+        app_model.tenant,
+        app_model,
+        serialize_runtime_site(site, features),
+        end_user_id,
+        features.can_replace_logo,
+    )
     return cast(dict[str, Any], marshal(app_site_info, AppSiteApi.app_fields))

api/core/app/app_config/common/sensitive_word_avoidance/manager.py

@@ -1,10 +1,73 @@
 from collections.abc import Mapping
-from typing import Any
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
 
 from core.app.app_config.entities import SensitiveWordAvoidanceEntity
 from core.moderation.factory import ModerationFactory
 
 
+class SensitiveWordAvoidanceDisabledConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: Literal[False] = False
+
+
+class SensitiveWordAvoidanceKeywordsConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: Literal[True] = True
+    type: Literal["keywords"]
+    config: dict[str, Any] = Field(default_factory=dict)
+
+    def run_provider_validation(self, tenant_id: str) -> None:
+        ModerationFactory.validate_config(name=self.type, tenant_id=tenant_id, config=self.config)
+
+
+class SensitiveWordAvoidanceOpenAIConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: Literal[True] = True
+    type: Literal["openai_moderation"]
+    config: dict[str, Any] = Field(default_factory=dict)
+
+    def run_provider_validation(self, tenant_id: str) -> None:
+        ModerationFactory.validate_config(name=self.type, tenant_id=tenant_id, config=self.config)
+
+
+class SensitiveWordAvoidanceAPIConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    enabled: Literal[True] = True
+    type: Literal["api"]
+    config: dict[str, Any] = Field(default_factory=dict)
+
+    def run_provider_validation(self, tenant_id: str) -> None:
+        ModerationFactory.validate_config(name=self.type, tenant_id=tenant_id, config=self.config)
+
+
+EnabledSensitiveWordAvoidanceConfig = Annotated[
+    SensitiveWordAvoidanceKeywordsConfig | SensitiveWordAvoidanceOpenAIConfig | SensitiveWordAvoidanceAPIConfig,
+    Field(discriminator="type"),
+]
+
+SensitiveWordAvoidanceConfig = Annotated[
+    SensitiveWordAvoidanceDisabledConfig | EnabledSensitiveWordAvoidanceConfig,
+    Field(discriminator="enabled"),
+]
+
+_sensitive_word_avoidance_adapter: TypeAdapter[SensitiveWordAvoidanceConfig] = TypeAdapter(SensitiveWordAvoidanceConfig)
+
+
+def _normalize_raw(raw: Any) -> Any:
+    if isinstance(raw, dict):
+        if raw.get("enabled") is None:
+            raw = {**raw, "enabled": False}
+        elif raw.get("enabled") is True and raw.get("config") is None:
+            raw = {**raw, "config": {}}
+    return raw
+
+
 class SensitiveWordAvoidanceConfigManager:
     @classmethod
     def convert(cls, config: Mapping[str, Any]) -> SensitiveWordAvoidanceEntity | None:
@@ -24,30 +87,24 @@ class SensitiveWordAvoidanceConfigManager:
     def validate_and_set_defaults(
         cls, tenant_id: str, config: dict[str, Any], only_structure_validate: bool = False
     ) -> tuple[dict[str, Any], list[str]]:
-        if not config.get("sensitive_word_avoidance"):
-            config["sensitive_word_avoidance"] = {"enabled": False}
-
-        if not isinstance(config["sensitive_word_avoidance"], dict):
+        raw = config.get("sensitive_word_avoidance") or {"enabled": False}
+        if not isinstance(raw, dict):
             raise ValueError("sensitive_word_avoidance must be of dict type")
 
-        if "enabled" not in config["sensitive_word_avoidance"] or not config["sensitive_word_avoidance"]["enabled"]:
-            config["sensitive_word_avoidance"]["enabled"] = False
+        try:
+            validated = _sensitive_word_avoidance_adapter.validate_python(_normalize_raw(raw))
+        except ValidationError:
+            raise
 
-        if config["sensitive_word_avoidance"]["enabled"]:
-            if not config["sensitive_word_avoidance"].get("type"):
-                raise ValueError("sensitive_word_avoidance.type is required")
-
-            if not only_structure_validate:
-                typ = config["sensitive_word_avoidance"]["type"]
-                if not isinstance(typ, str):
-                    raise ValueError("sensitive_word_avoidance.type must be a string")
-
-                sensitive_word_avoidance_config = config["sensitive_word_avoidance"].get("config")
-                if sensitive_word_avoidance_config is None:
-                    sensitive_word_avoidance_config = {}
-                if not isinstance(sensitive_word_avoidance_config, dict):
-                    raise ValueError("sensitive_word_avoidance.config must be a dict")
-
-                ModerationFactory.validate_config(name=typ, tenant_id=tenant_id, config=sensitive_word_avoidance_config)
+        if not only_structure_validate and isinstance(
+            validated,
+            (
+                SensitiveWordAvoidanceKeywordsConfig,
+                SensitiveWordAvoidanceOpenAIConfig,
+                SensitiveWordAvoidanceAPIConfig,
+            ),
+        ):
+            validated.run_provider_validation(tenant_id)
 
+        config["sensitive_word_avoidance"] = validated.model_dump()
         return config, ["sensitive_word_avoidance"]

api/core/app/app_config/easy_ui_based_app/dataset/manager.py

@@ -9,6 +9,7 @@ from core.app.app_config.entities import (
 )
 from core.entities.agent_entities import PlanningStrategy
 from core.rag.data_post_processor.data_post_processor import RerankingModelDict, WeightsDict
+from extensions.ext_database import db
 from models.model import AppMode, AppModelConfigDict
 from services.dataset_service import DatasetService
 
@@ -256,7 +257,7 @@ class DatasetConfigManager:
     @classmethod
     def is_dataset_exists(cls, tenant_id: str, dataset_id: str) -> bool:
         # verify if the dataset ID exists
-        dataset = DatasetService.get_dataset(dataset_id)
+        dataset = DatasetService.get_dataset(dataset_id, db.session)
 
         if not dataset:
             return False

api/core/app/apps/agent_app/app_generator.py

@@ -1,22 +1,29 @@
-"""Agent App generator: orchestrate one conversation turn for an Agent App.
+"""Agent App generator: orchestrate Agent App chat and finalize executions.
 
-Mirrors the agent_chat generator (conversation + message + queue + streamed
-response over the EasyUI chat pipeline), but the backing config comes from the
-bound Agent Soul and the answer is produced by ``AgentAppRunner`` calling the
-dify-agent backend rather than an in-process LLM/ReAct loop.
+The primary mode mirrors the agent_chat generator (conversation + message +
+queue + streamed response over the EasyUI chat pipeline), but the backing
+config comes from the bound Agent Soul and the answer is produced by
+``AgentAppRunner`` calling the dify-agent backend rather than an in-process
+LLM/ReAct loop.
+
+It also exposes a stateless build-finalize mode that reuses existing runtime
+context from the bound debug conversation, triggers the Agent backend side
+effect synchronously, and skips Dify-side chat/message persistence.
 """
 
 from __future__ import annotations
 
 import contextvars
+import json
 import logging
 import threading
 import uuid
-from collections.abc import Generator, Mapping
-from typing import Any
+from collections.abc import Generator, Mapping, Sequence
+from typing import Any, Literal
 
 from flask import Flask, current_app
-from sqlalchemy import select
+from pydantic import JsonValue
+from sqlalchemy import and_, or_, select
 
 from clients.agent_backend import AgentBackendRunEventAdapter
 from clients.agent_backend.factory import create_agent_backend_run_client
@@ -42,7 +49,15 @@ from core.app.llm.model_access import build_dify_model_access
 from core.ops.ops_trace_manager import TraceQueueManager
 from extensions.ext_database import db
 from models import Account, App, EndUser, Message
-from models.agent import Agent, AgentConfigSnapshot, AgentScope, AgentSource, AgentStatus
+from models.agent import (
+    Agent,
+    AgentConfigDraft,
+    AgentConfigDraftType,
+    AgentConfigSnapshot,
+    AgentScope,
+    AgentSource,
+    AgentStatus,
+)
 from models.agent_config_entities import AgentSoulConfig
 from services.conversation_service import ConversationService
 
@@ -53,6 +68,13 @@ class AgentAppGeneratorError(ValueError):
     """Raised when an Agent App turn cannot be set up."""
 
 
+def _append_prompt_file_mappings(query: str, prompt_file_mappings: Sequence[JsonValue]) -> str:
+    """Append raw request file references to the backend user prompt."""
+    if not prompt_file_mappings:
+        return query
+    return f"{query}\n{json.dumps(list(prompt_file_mappings), ensure_ascii=False)}"
+
+
 class AgentAppGenerator(MessageBasedAppGenerator):
     def generate(
         self,
@@ -66,17 +88,20 @@ class AgentAppGenerator(MessageBasedAppGenerator):
         if not streaming:
             raise AgentAppGeneratorError("Agent App only supports streaming mode")
 
-        query = args.get("query")
-        if not isinstance(query, str) or not query.strip():
-            raise AgentAppGeneratorError("query is required")
-        query = query.replace("\x00", "")
+        query = self._require_query(args)
         inputs = args["inputs"]
+        prompt_file_mappings = args.get("files") or []
 
         # Resolve the bound roster Agent + its current Agent Soul snapshot.
-        agent, snapshot, agent_soul = self._resolve_agent(app_model)
+        agent, agent_config_id, agent_config_version_kind, agent_soul = self._resolve_agent(
+            app_model,
+            invoke_from=invoke_from,
+            draft_type=args.get("draft_type"),
+            user=user,
+        )
         runtime_session_snapshot_id = self._runtime_session_snapshot_id(
             invoke_from=invoke_from,
-            snapshot_id=snapshot.id,
+            snapshot_id=agent_config_id,
         )
 
         conversation = None
@@ -109,6 +134,7 @@ class AgentAppGenerator(MessageBasedAppGenerator):
             ),
             query=query,
             files=[],
+            prompt_file_mappings=prompt_file_mappings,
             parent_message_id=(
                 args.get("parent_message_id")
                 if invoke_from not in {InvokeFrom.SERVICE_API, InvokeFrom.OPENAPI}
@@ -123,7 +149,8 @@ class AgentAppGenerator(MessageBasedAppGenerator):
             call_depth=0,
             trace_manager=trace_manager,
             agent_id=agent.id,
-            agent_config_snapshot_id=snapshot.id,
+            agent_config_snapshot_id=agent_config_id,
+            agent_config_version_kind=agent_config_version_kind,
             agent_runtime_session_snapshot_id=runtime_session_snapshot_id,
         )
 
@@ -163,6 +190,86 @@ class AgentAppGenerator(MessageBasedAppGenerator):
         )
         return AgentAppGenerateResponseConverter.convert(response=response, invoke_from=invoke_from)
 
+    def generate_stateless(
+        self,
+        *,
+        app_model: App,
+        user: Account | EndUser,
+        args: Mapping[str, Any],
+        invoke_from: InvokeFrom,
+    ) -> Mapping[str, Any]:
+        """Run one Agent App turn without persisting Dify conversation messages."""
+        query = self._require_query(args)
+        conversation_id = args.get("conversation_id")
+        if not isinstance(conversation_id, str) or not conversation_id:
+            raise AgentAppGeneratorError("conversation_id is required")
+
+        agent, agent_config_id, agent_config_version_kind, agent_soul = self._resolve_agent(
+            app_model,
+            invoke_from=invoke_from,
+            draft_type=args.get("draft_type"),
+            user=user,
+        )
+        runtime_session_snapshot_id = self._runtime_session_snapshot_id(
+            invoke_from=invoke_from,
+            snapshot_id=agent_config_id,
+        )
+
+        return self._run_stateless(
+            app_model=app_model,
+            user=user,
+            invoke_from=invoke_from,
+            query=query,
+            conversation_id=conversation_id,
+            agent=agent,
+            agent_config_id=agent_config_id,
+            agent_config_version_kind=agent_config_version_kind,
+            agent_soul=agent_soul,
+            runtime_session_snapshot_id=runtime_session_snapshot_id,
+        )
+
+    def _run_stateless(
+        self,
+        *,
+        app_model: App,
+        user: Account | EndUser,
+        invoke_from: InvokeFrom,
+        query: str,
+        conversation_id: str,
+        agent: Agent,
+        agent_config_id: str,
+        agent_config_version_kind: Literal["snapshot", "draft", "build_draft"],
+        agent_soul: AgentSoulConfig,
+        runtime_session_snapshot_id: str | None,
+    ) -> Mapping[str, Any]:
+        """Run the Agent backend without creating or updating Dify chat records.
+
+        Build-chat finalization is an action against the Agent backend (for
+        example, ``dify-agent config push``). It may reuse the active build-chat
+        runtime snapshot for shell/config context, but the API side must not add
+        a synthetic user/assistant turn to the debug conversation.
+        """
+
+        dify_context = DifyRunContext(
+            tenant_id=app_model.tenant_id,
+            app_id=app_model.id,
+            user_id=user.id,
+            user_from=UserFrom.ACCOUNT if isinstance(user, Account) else UserFrom.END_USER,
+            invoke_from=invoke_from,
+        )
+        self._build_runner(dify_context).run_stateless(
+            dify_context=dify_context,
+            agent_id=agent.id,
+            agent_config_snapshot_id=agent_config_id,
+            agent_config_version_kind=agent_config_version_kind,
+            agent_soul=agent_soul,
+            conversation_id=conversation_id,
+            query=query,
+            idempotency_key=str(uuid.uuid4()),
+            session_scope_snapshot_id=runtime_session_snapshot_id,
+        )
+        return {"result": "success"}
+
     def resume_after_form_submission(
         self,
         *,
@@ -179,10 +286,15 @@ class AgentAppGenerator(MessageBasedAppGenerator):
         persisted to the conversation. Live streaming to a reconnected client is
         out of scope here — the message is persisted and can be re-fetched.
         """
-        agent, snapshot, agent_soul = self._resolve_agent(app_model)
         conversation = ConversationService.get_conversation(
             app_model=app_model, conversation_id=conversation_id, user=user
         )
+        agent, agent_config_id, agent_config_version_kind, agent_soul = self._resolve_agent(
+            app_model,
+            invoke_from=invoke_from,
+            draft_type=self._resume_draft_type(app_model=app_model, conversation=conversation, user=user),
+            user=user,
+        )
 
         app_config = AgentAppConfigManager.get_app_config(
             app_model=app_model,
@@ -226,7 +338,8 @@ class AgentAppGenerator(MessageBasedAppGenerator):
             call_depth=0,
             trace_manager=trace_manager,
             agent_id=agent.id,
-            agent_config_snapshot_id=snapshot.id,
+            agent_config_snapshot_id=agent_config_id,
+            agent_config_version_kind=agent_config_version_kind,
         )
 
         conversation, message = self._init_generate_records(application_generate_entity, conversation)
@@ -267,6 +380,30 @@ class AgentAppGenerator(MessageBasedAppGenerator):
             stream=False,
         )
 
+    @staticmethod
+    def _resume_draft_type(*, app_model: App, conversation: Any, user: Account | EndUser) -> str | None:
+        if conversation.invoke_from != InvokeFrom.DEBUGGER:
+            return None
+        active_session = AgentAppRuntimeSessionStore().load_active_session_for_conversation(
+            tenant_id=app_model.tenant_id,
+            app_id=app_model.id,
+            conversation_id=conversation.id,
+        )
+        snapshot_id = active_session.scope.agent_config_snapshot_id if active_session is not None else None
+        if snapshot_id and isinstance(user, Account):
+            draft = db.session.scalar(
+                select(AgentConfigDraft).where(
+                    AgentConfigDraft.tenant_id == app_model.tenant_id,
+                    AgentConfigDraft.id == snapshot_id,
+                )
+            )
+            if draft is not None:
+                if draft.draft_type == AgentConfigDraftType.DEBUG_BUILD and draft.account_id == user.id:
+                    return AgentConfigDraftType.DEBUG_BUILD.value
+                if draft.draft_type == AgentConfigDraftType.DRAFT and draft.account_id is None:
+                    return AgentConfigDraftType.DRAFT.value
+        return AgentConfigDraftType.DRAFT.value
+
     def _generate_worker(
         self,
         *,
@@ -311,6 +448,10 @@ class AgentAppGenerator(MessageBasedAppGenerator):
                     )
                     if handled:
                         return
+                    query = _append_prompt_file_mappings(
+                        query=query,
+                        prompt_file_mappings=application_generate_entity.prompt_file_mappings,
+                    )
 
                 dify_context = DifyRunContext(
                     tenant_id=app_config.tenant_id,
@@ -319,27 +460,18 @@ class AgentAppGenerator(MessageBasedAppGenerator):
                     user_from=user_from,
                     invoke_from=application_generate_entity.invoke_from,
                 )
-                credentials_provider, _ = build_dify_model_access(dify_context)
                 _, _, agent_soul = self._resolve_agent_by_id(
                     tenant_id=app_config.tenant_id,
                     agent_id=application_generate_entity.agent_id,
                     snapshot_id=application_generate_entity.agent_config_snapshot_id,
                 )
 
-                runner = AgentAppRunner(
-                    request_builder=AgentAppRuntimeRequestBuilder(credentials_provider=credentials_provider),
-                    agent_backend_client=create_agent_backend_run_client(
-                        base_url=dify_config.AGENT_BACKEND_BASE_URL,
-                        use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
-                        fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
-                    ),
-                    event_adapter=AgentBackendRunEventAdapter(),
-                    session_store=AgentAppRuntimeSessionStore(),
-                )
+                runner = self._build_runner(dify_context)
                 runner.run(
                     dify_context=dify_context,
                     agent_id=application_generate_entity.agent_id,
                     agent_config_snapshot_id=application_generate_entity.agent_config_snapshot_id,
+                    agent_config_version_kind=application_generate_entity.agent_config_version_kind,
                     agent_soul=agent_soul,
                     conversation_id=conversation.id,
                     query=query,
@@ -356,6 +488,27 @@ class AgentAppGenerator(MessageBasedAppGenerator):
             finally:
                 db.session.close()
 
+    @staticmethod
+    def _require_query(args: Mapping[str, Any]) -> str:
+        query = args.get("query")
+        if not isinstance(query, str) or not query.strip():
+            raise AgentAppGeneratorError("query is required")
+        return query.replace("\x00", "")
+
+    @staticmethod
+    def _build_runner(dify_context: DifyRunContext) -> AgentAppRunner:
+        credentials_provider, _ = build_dify_model_access(dify_context)
+        return AgentAppRunner(
+            request_builder=AgentAppRuntimeRequestBuilder(credentials_provider=credentials_provider),
+            agent_backend_client=create_agent_backend_run_client(
+                base_url=dify_config.AGENT_BACKEND_BASE_URL,
+                use_fake=dify_config.AGENT_BACKEND_USE_FAKE,
+                fake_scenario=dify_config.AGENT_BACKEND_FAKE_SCENARIO,
+            ),
+            event_adapter=AgentBackendRunEventAdapter(),
+            session_store=AgentAppRuntimeSessionStore(),
+        )
+
     def _run_input_guards(
         self,
         *,
@@ -421,50 +574,138 @@ class AgentAppGenerator(MessageBasedAppGenerator):
 
         return False, query
 
-    def _resolve_agent(self, app_model: App) -> tuple[Agent, AgentConfigSnapshot, AgentSoulConfig]:
+    def _resolve_agent(
+        self,
+        app_model: App,
+        *,
+        invoke_from: InvokeFrom,
+        draft_type: Any,
+        user: Account | EndUser,
+    ) -> tuple[Agent, str, Literal["snapshot", "draft", "build_draft"], AgentSoulConfig]:
         agent = db.session.scalar(
-            select(Agent).where(
-                Agent.app_id == app_model.id,
-                Agent.scope == AgentScope.ROSTER,
-                Agent.source == AgentSource.AGENT_APP,
+            select(Agent)
+            .where(
+                Agent.tenant_id == app_model.tenant_id,
                 Agent.status == AgentStatus.ACTIVE,
+                or_(
+                    and_(
+                        Agent.app_id == app_model.id,
+                        Agent.scope == AgentScope.ROSTER,
+                        Agent.source == AgentSource.AGENT_APP,
+                    ),
+                    Agent.backing_app_id == app_model.id,
+                ),
             )
+            .order_by(Agent.created_at.desc())
+            .limit(1)
         )
         if agent is None:
             raise AgentAppGeneratorError("Agent App has no bound Agent")
-        return self._resolve_agent_by_id(
-            tenant_id=app_model.tenant_id, agent_id=agent.id, snapshot_id=agent.active_config_snapshot_id
+        if invoke_from == InvokeFrom.DEBUGGER:
+            draft = self._resolve_debug_draft(
+                tenant_id=app_model.tenant_id,
+                agent=agent,
+                draft_type=draft_type,
+                account_id=user.id if isinstance(user, Account) else None,
+            )
+            agent_soul = AgentSoulConfig.model_validate(draft.config_snapshot_dict)
+            config_version_kind: Literal["snapshot", "draft", "build_draft"] = (
+                "build_draft" if draft.draft_type == AgentConfigDraftType.DEBUG_BUILD else "draft"
+            )
+            return agent, draft.id, config_version_kind, agent_soul
+        _, snapshot, agent_soul = self._resolve_agent_by_id(
+            tenant_id=app_model.tenant_id,
+            agent_id=agent.id,
+            snapshot_id=agent.active_config_snapshot_id,
         )
+        return agent, snapshot.id, "snapshot", agent_soul
 
     @staticmethod
     def _runtime_session_snapshot_id(*, invoke_from: InvokeFrom, snapshot_id: str) -> str | None:
         """Return the session scope snapshot id for Agent App runtime state.
 
-        Console preview/debug chat is an editing workspace: saving Agent Soul
-        creates replacement snapshots, but the user expects the same preview
-        conversation to keep context while trying prompt changes. Use a stable
-        NULL snapshot scope for debugger runs so each turn can use the latest
-        Agent Soul while reusing the conversation history. Published/web/API
-        runs keep snapshot-scoped sessions for reproducible runtime state.
+        Console preview/debug chat uses a stable Agent draft row id; build mode
+        uses the current user's build-draft row id. Published/web/API runs use
+        immutable published snapshot ids. This keeps runtime session continuity
+        inside one editable surface without mixing draft/build/published state.
         """
-        if invoke_from == InvokeFrom.DEBUGGER:
-            return None
         return snapshot_id
 
+    @staticmethod
+    def _resolve_debug_draft(
+        *, tenant_id: str, agent: Agent, draft_type: Any, account_id: str | None
+    ) -> AgentConfigDraft:
+        effective_draft_type = (
+            AgentConfigDraftType.DEBUG_BUILD
+            if draft_type == AgentConfigDraftType.DEBUG_BUILD.value
+            else AgentConfigDraftType.DRAFT
+        )
+        stmt = select(AgentConfigDraft).where(
+            AgentConfigDraft.tenant_id == tenant_id,
+            AgentConfigDraft.agent_id == agent.id,
+            AgentConfigDraft.draft_type == effective_draft_type,
+        )
+        if effective_draft_type == AgentConfigDraftType.DEBUG_BUILD:
+            if not account_id:
+                raise AgentAppGeneratorError("Build draft requires an account user")
+            stmt = stmt.where(AgentConfigDraft.account_id == account_id)
+        else:
+            stmt = stmt.where(AgentConfigDraft.account_id.is_(None))
+        draft = db.session.scalar(stmt.order_by(AgentConfigDraft.updated_at.desc()).limit(1))
+        if draft is not None:
+            return draft
+        if effective_draft_type == AgentConfigDraftType.DEBUG_BUILD:
+            raise AgentAppGeneratorError("Agent build draft not found")
+        _, snapshot, agent_soul = AgentAppGenerator._resolve_agent_by_id(
+            tenant_id=tenant_id,
+            agent_id=agent.id,
+            snapshot_id=agent.active_config_snapshot_id,
+        )
+        draft = AgentConfigDraft(
+            tenant_id=tenant_id,
+            agent_id=agent.id,
+            draft_type=AgentConfigDraftType.DRAFT,
+            account_id=None,
+            draft_owner_key="",
+            base_snapshot_id=snapshot.id,
+            config_snapshot=agent_soul,
+            created_by=agent.created_by,
+            updated_by=agent.updated_by,
+        )
+        db.session.add(draft)
+        db.session.flush()
+        return draft
+
     @staticmethod
     def _resolve_agent_by_id(
         *, tenant_id: str, agent_id: str, snapshot_id: str | None
-    ) -> tuple[Agent, AgentConfigSnapshot, AgentSoulConfig]:
+    ) -> tuple[Agent, AgentConfigSnapshot | AgentConfigDraft, AgentSoulConfig]:
         agent = db.session.scalar(select(Agent).where(Agent.id == agent_id, Agent.tenant_id == tenant_id))
         if agent is None:
             raise AgentAppGeneratorError("Agent not found")
         if not snapshot_id:
             raise AgentAppGeneratorError("Agent has no published version")
-        snapshot = db.session.scalar(select(AgentConfigSnapshot).where(AgentConfigSnapshot.id == snapshot_id))
-        if snapshot is None:
+        snapshot = db.session.scalar(
+            select(AgentConfigSnapshot).where(
+                AgentConfigSnapshot.tenant_id == tenant_id,
+                AgentConfigSnapshot.agent_id == agent_id,
+                AgentConfigSnapshot.id == snapshot_id,
+            )
+        )
+        if snapshot is not None:
+            agent_soul = AgentSoulConfig.model_validate(snapshot.config_snapshot_dict)
+            return agent, snapshot, agent_soul
+        draft = db.session.scalar(
+            select(AgentConfigDraft).where(
+                AgentConfigDraft.tenant_id == tenant_id,
+                AgentConfigDraft.agent_id == agent_id,
+                AgentConfigDraft.id == snapshot_id,
+            )
+        )
+        if draft is None:
             raise AgentAppGeneratorError("Agent published version not found")
-        agent_soul = AgentSoulConfig.model_validate(snapshot.config_snapshot_dict)
-        return agent, snapshot, agent_soul
+        agent_soul = AgentSoulConfig.model_validate(draft.config_snapshot_dict)
+        return agent, draft, agent_soul
 
 
 __all__ = ["AgentAppGenerator", "AgentAppGeneratorError"]

api/core/app/apps/agent_app/app_runner.py

@@ -1,18 +1,23 @@
-"""Agent App runner: drive one conversation turn through the dify-agent backend.
+"""Agent App runner: drive Agent backend turns for both chat and finalize flows.
 
 Unlike the legacy ``AgentChatAppRunner`` (which runs an in-process ReAct loop),
-this runner delegates to the Agent backend: build the run request from the
-Agent Soul + conversation, create the run, consume its event stream, and
-republish the assistant answer as chat queue events so the existing
-EasyUI chat task pipeline persists the message and streams SSE. The conversation
-``session_snapshot`` is saved on success for multi-turn continuity (S3).
+this runner delegates to the Agent backend and supports two execution modes.
+
+- Normal chat turns build the run request from the Agent Soul + conversation,
+  consume backend stream events, republish the assistant answer through the
+  existing EasyUI chat task pipeline, and save the conversation
+  ``session_snapshot`` on success for multi-turn continuity (S3).
+- Stateless build-finalize turns reuse any prior conversation snapshot only to
+  construct the backend request, wait synchronously for backend completion, and
+  intentionally do not persist Dify-side chat records or runtime-session state.
 """
 
 from __future__ import annotations
 
 import json
 import logging
-from typing import Any
+from decimal import Decimal
+from typing import Any, Literal
 
 from dify_agent.layers.ask_human import AskHumanToolArgs
 from dify_agent.protocol import DeferredToolResultsPayload
@@ -28,6 +33,7 @@ from clients.agent_backend import (
     AgentBackendStreamInternalEvent,
     extract_runtime_layer_specs,
 )
+from configs import dify_config
 from core.app.apps.agent_app.runtime_request_builder import (
     AgentAppRuntimeBuildContext,
     AgentAppRuntimeRequest,
@@ -41,13 +47,16 @@ from core.app.apps.agent_app.session_store import (
 from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
 from core.app.apps.exc import GenerateTaskStoppedError
 from core.app.entities.app_invoke_entities import DifyRunContext
-from core.app.entities.queue_entities import QueueLLMChunkEvent, QueueMessageEndEvent
+from core.app.entities.queue_entities import QueueAgentThoughtEvent, QueueLLMChunkEvent, QueueMessageEndEvent
 from core.repositories.human_input_repository import HumanInputFormRepository, HumanInputFormRepositoryImpl
 from core.workflow.nodes.agent_v2.ask_human_hitl import AskHumanFormBuildError, create_ask_human_form
 from core.workflow.nodes.agent_v2.ask_human_resume import build_deferred_tool_results, resolve_ask_human_form
+from extensions.ext_database import db
 from graphon.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
 from graphon.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage
 from models.agent_config_entities import AgentSoulConfig
+from models.enums import CreatorUserRole
+from models.model import MessageAgentThought
 
 logger = logging.getLogger(__name__)
 
@@ -134,6 +143,283 @@ def publish_message_end(
     )
 
 
+class _AgentProcessRecorder:
+    """Persist Agent v2 thinking/tool process events through the legacy thought model."""
+
+    def __init__(
+        self,
+        *,
+        dify_context: DifyRunContext,
+        message_id: str,
+        queue_manager: AppQueueManager,
+    ) -> None:
+        self._dify_context = dify_context
+        self._message_id = message_id
+        self._queue_manager = queue_manager
+        self._next_position = 1
+        self._thinking_by_index: dict[int, str] = {}
+        self._tool_by_index: dict[int, str] = {}
+        self._tool_by_call_id: dict[str, str] = {}
+        self._open_tool_by_name: dict[str, set[str]] = {}
+
+    def handle_stream_event(self, event: AgentBackendStreamInternalEvent) -> None:
+        data = event.data
+        if not isinstance(data, dict):
+            return
+
+        event_kind = data.get("event_kind")
+        if event_kind == "part_delta":
+            self._handle_part_delta(data)
+        elif event_kind == "part_start":
+            self._handle_part(data)
+        elif event_kind in {"function_tool_call", "output_tool_call"}:
+            self._handle_tool_call_event(data)
+        elif event_kind in {"function_tool_result", "output_tool_result"}:
+            self._handle_tool_result_event(data)
+
+    def _handle_part_delta(self, data: dict[str, Any]) -> None:
+        delta = data.get("delta")
+        if not isinstance(delta, dict):
+            return
+
+        index = _event_index(data)
+        delta_kind = delta.get("part_delta_kind")
+        if delta_kind == "thinking":
+            content_delta = delta.get("content_delta")
+            if isinstance(content_delta, str) and content_delta:
+                self._append_thinking(index, content_delta)
+            return
+
+        if delta_kind == "tool_call":
+            self._record_tool_call_delta(index, delta)
+
+    def _handle_part(self, data: dict[str, Any]) -> None:
+        part = data.get("part")
+        if not isinstance(part, dict):
+            return
+
+        index = _event_index(data)
+        part_kind = part.get("part_kind")
+        if part_kind == "thinking":
+            content = part.get("content")
+            if isinstance(content, str) and content:
+                self._append_thinking(index, content)
+            return
+
+        if part_kind in {"tool-call", "builtin-tool-call"}:
+            self._record_tool_call_part(index, part)
+            return
+
+        if part_kind in {"tool-return", "builtin-tool-return"}:
+            self._record_tool_return_part(part)
+
+    def _handle_tool_call_event(self, data: dict[str, Any]) -> None:
+        part = data.get("part")
+        if isinstance(part, dict):
+            self._record_tool_call_part(_event_index(data), part)
+
+    def _handle_tool_result_event(self, data: dict[str, Any]) -> None:
+        part = data.get("part") or data.get("result")
+        if isinstance(part, dict):
+            self._record_tool_return_part(part)
+            return
+
+        content = data.get("content")
+        if content is not None:
+            self._record_tool_observation(
+                tool_call_id=_string_or_none(data.get("tool_call_id")),
+                tool_name=_string_or_none(data.get("tool_name")),
+                observation=content,
+            )
+
+    def _append_thinking(self, index: int, content_delta: str) -> None:
+        thought_id = self._thinking_by_index.get(index)
+        if thought_id is None:
+            thought_id = self._create_thought(thought=content_delta)
+            self._thinking_by_index[index] = thought_id
+            return
+        self._update_thought(thought_id, thought_delta=content_delta)
+
+    def _record_tool_call_delta(self, index: int, delta: dict[str, Any]) -> None:
+        tool_call_id = _string_or_none(delta.get("tool_call_id"))
+        tool_name = _string_or_none(delta.get("tool_name_delta"))
+        args_delta = delta.get("args_delta")
+        thought_id = self._lookup_tool_thought(index=index, tool_call_id=tool_call_id)
+        if thought_id is None:
+            thought_id = self._create_thought(tool=tool_name, tool_input=_json_or_text(args_delta))
+            self._remember_tool_thought(
+                index=index, tool_call_id=tool_call_id, tool_name=tool_name, thought_id=thought_id
+            )
+            return
+
+        self._update_thought(
+            thought_id,
+            tool=tool_name,
+            tool_input_delta=_json_or_text(args_delta),
+        )
+
+    def _record_tool_call_part(self, index: int, part: dict[str, Any]) -> None:
+        tool_call_id = _string_or_none(part.get("tool_call_id"))
+        tool_name = _string_or_none(part.get("tool_name"))
+        thought_id = self._lookup_tool_thought(index=index, tool_call_id=tool_call_id)
+        if thought_id is None:
+            thought_id = self._create_thought(tool=tool_name, tool_input=_json_or_text(part.get("args")))
+            self._remember_tool_thought(
+                index=index, tool_call_id=tool_call_id, tool_name=tool_name, thought_id=thought_id
+            )
+            return
+
+        self._update_thought(
+            thought_id,
+            tool=tool_name,
+            tool_input=_json_or_text(part.get("args")),
+        )
+
+    def _record_tool_return_part(self, part: dict[str, Any]) -> None:
+        tool_call_id = _string_or_none(part.get("tool_call_id"))
+        tool_name = _string_or_none(part.get("tool_name"))
+        content = part.get("content")
+        if content is None:
+            content = part
+        self._record_tool_observation(tool_call_id=tool_call_id, tool_name=tool_name, observation=content)
+
+    def _record_tool_observation(self, *, tool_call_id: str | None, tool_name: str | None, observation: Any) -> None:
+        thought_id = self._lookup_observation_thought(tool_call_id=tool_call_id, tool_name=tool_name)
+        if thought_id is None:
+            thought_id = self._create_thought(tool=tool_name)
+        else:
+            self._mark_tool_observed(thought_id)
+        self._update_thought(thought_id, observation=_json_or_text(observation))
+
+    def _lookup_tool_thought(self, *, index: int, tool_call_id: str | None) -> str | None:
+        if tool_call_id and tool_call_id in self._tool_by_call_id:
+            return self._tool_by_call_id[tool_call_id]
+        return self._tool_by_index.get(index)
+
+    def _remember_tool_thought(
+        self, *, index: int, tool_call_id: str | None, tool_name: str | None, thought_id: str
+    ) -> None:
+        self._tool_by_index[index] = thought_id
+        if tool_call_id:
+            self._tool_by_call_id[tool_call_id] = thought_id
+        if tool_name:
+            self._open_tool_by_name.setdefault(tool_name, set()).add(thought_id)
+
+    def _lookup_observation_thought(self, *, tool_call_id: str | None, tool_name: str | None) -> str | None:
+        if tool_call_id:
+            return self._tool_by_call_id.get(tool_call_id)
+        if tool_name:
+            open_thought_ids = self._open_tool_by_name.get(tool_name, set())
+            if len(open_thought_ids) == 1:
+                return next(iter(open_thought_ids))
+        return None
+
+    def _mark_tool_observed(self, thought_id: str) -> None:
+        for open_thought_ids in self._open_tool_by_name.values():
+            open_thought_ids.discard(thought_id)
+
+    def _create_thought(
+        self, *, thought: str | None = None, tool: str | None = None, tool_input: str | None = None
+    ) -> str:
+        row = MessageAgentThought(
+            message_id=self._message_id,
+            message_chain_id=None,
+            thought=thought,
+            tool=tool,
+            tool_labels_str=_tool_labels(tool),
+            tool_meta_str="{}",
+            tool_input=tool_input,
+            observation=None,
+            tool_process_data=None,
+            message=None,
+            message_token=0,
+            message_unit_price=Decimal(0),
+            message_price_unit=Decimal("0.001"),
+            message_files="",
+            answer="",
+            answer_token=0,
+            answer_unit_price=Decimal(0),
+            answer_price_unit=Decimal("0.001"),
+            tokens=0,
+            total_price=Decimal(0),
+            position=self._next_position,
+            currency="USD",
+            latency=0,
+            created_by_role=self._created_by_role(),
+            created_by=self._dify_context.user_id,
+        )
+        self._next_position += 1
+        db.session.add(row)
+        db.session.commit()
+        thought_id = str(row.id)
+        self._queue_manager.publish(
+            QueueAgentThoughtEvent(agent_thought_id=thought_id), PublishFrom.APPLICATION_MANAGER
+        )
+        return thought_id
+
+    def _update_thought(
+        self,
+        thought_id: str,
+        *,
+        thought_delta: str | None = None,
+        tool: str | None = None,
+        tool_input: str | None = None,
+        tool_input_delta: str | None = None,
+        observation: str | None = None,
+    ) -> None:
+        row = db.session.get(MessageAgentThought, thought_id)
+        if row is None:
+            return
+
+        if thought_delta:
+            row.thought = f"{row.thought or ''}{thought_delta}"
+        if tool:
+            row.tool = tool
+            row.tool_labels_str = _tool_labels(tool)
+        if tool_input is not None:
+            row.tool_input = tool_input
+        if tool_input_delta:
+            row.tool_input = f"{row.tool_input or ''}{tool_input_delta}"
+        if observation is not None:
+            row.observation = observation
+
+        db.session.commit()
+        self._queue_manager.publish(
+            QueueAgentThoughtEvent(agent_thought_id=thought_id), PublishFrom.APPLICATION_MANAGER
+        )
+
+    def _created_by_role(self) -> CreatorUserRole:
+        if self._dify_context.invoke_from.runs_as_account():
+            return CreatorUserRole.ACCOUNT
+        return CreatorUserRole.END_USER
+
+
+def _event_index(data: dict[str, Any]) -> int:
+    index = data.get("index")
+    return index if isinstance(index, int) else -1
+
+
+def _string_or_none(value: Any) -> str | None:
+    return value if isinstance(value, str) and value else None
+
+
+def _json_or_text(value: Any) -> str | None:
+    if value is None:
+        return None
+    if isinstance(value, str):
+        return value
+    try:
+        return json.dumps(value, ensure_ascii=False)
+    except Exception:
+        return str(value)
+
+
+def _tool_labels(tool: str | None) -> str:
+    if not tool:
+        return "{}"
+    return json.dumps({tool: {"en_US": tool, "zh_Hans": tool}}, ensure_ascii=False)
+
+
 class AgentAppRunner:
     """Runs one Agent App conversation turn against the Agent backend."""
 
@@ -156,6 +442,7 @@ class AgentAppRunner:
         dify_context: DifyRunContext,
         agent_id: str,
         agent_config_snapshot_id: str,
+        agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot",
         agent_soul: AgentSoulConfig,
         conversation_id: str,
         query: str,
@@ -164,42 +451,34 @@ class AgentAppRunner:
         queue_manager: AppQueueManager,
         session_scope_snapshot_id: str | None | _DefaultSessionScopeSnapshotId = _DEFAULT_SESSION_SCOPE_SNAPSHOT_ID,
     ) -> None:
-        if isinstance(session_scope_snapshot_id, _DefaultSessionScopeSnapshotId):
-            effective_session_scope_snapshot_id: str | None = agent_config_snapshot_id
-        else:
-            effective_session_scope_snapshot_id = session_scope_snapshot_id
-        scope = AgentAppSessionScope(
-            tenant_id=dify_context.tenant_id,
-            app_id=dify_context.app_id,
-            conversation_id=conversation_id,
+        scope = self._build_session_scope(
+            dify_context=dify_context,
             agent_id=agent_id,
-            agent_config_snapshot_id=effective_session_scope_snapshot_id,
+            agent_config_snapshot_id=agent_config_snapshot_id,
+            conversation_id=conversation_id,
+            session_scope_snapshot_id=session_scope_snapshot_id,
         )
         # ENG-638: if a prior turn paused on ask_human and the form is now answered,
         # resume by threading the human's reply into this run as deferred_tool_results.
         stored = self._session_store.load_active_session(scope)
-        session_snapshot = stored.session_snapshot if stored is not None else None
-        deferred_tool_results = self._resolve_pending_ask_human(
-            stored=stored, dify_context=dify_context, message_id=message_id
-        )
-
-        runtime = self._request_builder.build(
-            AgentAppRuntimeBuildContext(
-                dify_context=dify_context,
-                agent_id=agent_id,
-                agent_config_snapshot_id=agent_config_snapshot_id,
-                agent_soul=agent_soul,
-                conversation_id=conversation_id,
-                user_query=query,
-                idempotency_key=message_id,
-                session_snapshot=session_snapshot,
-                deferred_tool_results=deferred_tool_results,
-            )
+        runtime = self._build_runtime(
+            dify_context=dify_context,
+            agent_id=agent_id,
+            agent_config_snapshot_id=agent_config_snapshot_id,
+            agent_config_version_kind=agent_config_version_kind,
+            agent_soul=agent_soul,
+            conversation_id=conversation_id,
+            query=query,
+            idempotency_key=message_id,
+            stored=stored,
+            message_id=message_id,
         )
 
         create_response = self._agent_backend_client.create_run(runtime.request)
         terminal, streamed_answer = self._consume_stream(
             create_response.run_id,
+            dify_context=dify_context,
+            message_id=message_id,
             queue_manager=queue_manager,
             model_name=model_name,
             query=query,
@@ -241,6 +520,111 @@ class AgentAppRunner:
             runtime_layer_specs=extract_runtime_layer_specs(runtime.request.composition),
         )
 
+    def run_stateless(
+        self,
+        *,
+        dify_context: DifyRunContext,
+        agent_id: str,
+        agent_config_snapshot_id: str,
+        agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot",
+        agent_soul: AgentSoulConfig,
+        conversation_id: str,
+        query: str,
+        idempotency_key: str,
+        session_scope_snapshot_id: str | None | _DefaultSessionScopeSnapshotId = _DEFAULT_SESSION_SCOPE_SNAPSHOT_ID,
+    ) -> None:
+        """Run the Agent backend without creating Dify chat message records.
+
+        This path is used by build-chat finalization: the API must trigger the
+        backend side effects in the existing conversation session, but it must
+        not persist a synthetic user/assistant turn, update API-side runtime
+        session rows, or set up HITL state that depends on one.
+        """
+        scope = self._build_session_scope(
+            dify_context=dify_context,
+            agent_id=agent_id,
+            agent_config_snapshot_id=agent_config_snapshot_id,
+            conversation_id=conversation_id,
+            session_scope_snapshot_id=session_scope_snapshot_id,
+        )
+        runtime = self._build_runtime(
+            dify_context=dify_context,
+            agent_id=agent_id,
+            agent_config_snapshot_id=agent_config_snapshot_id,
+            agent_config_version_kind=agent_config_version_kind,
+            agent_soul=agent_soul,
+            conversation_id=conversation_id,
+            query=query,
+            idempotency_key=idempotency_key,
+            stored=self._session_store.load_active_session(scope),
+            message_id=None,
+        )
+
+        create_response = self._agent_backend_client.create_run(runtime.request)
+        status = self._agent_backend_client.wait_run(
+            create_response.run_id,
+            timeout_seconds=dify_config.APP_MAX_EXECUTION_TIME,
+        )
+        if status.status != "succeeded":
+            error = getattr(status, "error", None) or f"Agent backend run ended with status {status.status}."
+            raise AgentBackendError(str(error))
+
+    def _build_session_scope(
+        self,
+        *,
+        dify_context: DifyRunContext,
+        agent_id: str,
+        agent_config_snapshot_id: str,
+        conversation_id: str,
+        session_scope_snapshot_id: str | None | _DefaultSessionScopeSnapshotId,
+    ) -> AgentAppSessionScope:
+        if isinstance(session_scope_snapshot_id, _DefaultSessionScopeSnapshotId):
+            effective_session_scope_snapshot_id: str | None = agent_config_snapshot_id
+        else:
+            effective_session_scope_snapshot_id = session_scope_snapshot_id
+        return AgentAppSessionScope(
+            tenant_id=dify_context.tenant_id,
+            app_id=dify_context.app_id,
+            conversation_id=conversation_id,
+            agent_id=agent_id,
+            agent_config_snapshot_id=effective_session_scope_snapshot_id,
+        )
+
+    def _build_runtime(
+        self,
+        *,
+        dify_context: DifyRunContext,
+        agent_id: str,
+        agent_config_snapshot_id: str,
+        agent_config_version_kind: Literal["snapshot", "draft", "build_draft"],
+        agent_soul: AgentSoulConfig,
+        conversation_id: str,
+        query: str,
+        idempotency_key: str,
+        stored: StoredAgentAppSession | None,
+        message_id: str | None,
+    ) -> AgentAppRuntimeRequest:
+        session_snapshot = stored.session_snapshot if stored is not None else None
+        deferred_tool_results = (
+            self._resolve_pending_ask_human(stored=stored, dify_context=dify_context, message_id=message_id)
+            if message_id is not None
+            else None
+        )
+        return self._request_builder.build(
+            AgentAppRuntimeBuildContext(
+                dify_context=dify_context,
+                agent_id=agent_id,
+                agent_config_snapshot_id=agent_config_snapshot_id,
+                agent_config_version_kind=agent_config_version_kind,
+                agent_soul=agent_soul,
+                conversation_id=conversation_id,
+                user_query=query,
+                idempotency_key=idempotency_key,
+                session_snapshot=session_snapshot,
+                deferred_tool_results=deferred_tool_results,
+            )
+        )
+
     def _pause_for_ask_human(
         self,
         *,
@@ -334,12 +718,19 @@ class AgentAppRunner:
         self,
         run_id: str,
         *,
+        dify_context: DifyRunContext,
+        message_id: str,
         queue_manager: AppQueueManager,
         model_name: str,
         query: str | None,
     ):
         terminal = None
         streamed_answer_parts: list[str] = []
+        process_recorder = _AgentProcessRecorder(
+            dify_context=dify_context,
+            message_id=message_id,
+            queue_manager=queue_manager,
+        )
         for public_event in self._agent_backend_client.stream_events(run_id):
             if queue_manager.is_stopped():
                 self._cancel_run(run_id)
@@ -353,6 +744,17 @@ class AgentAppRunner:
                     AgentBackendInternalEventType.STREAM_EVENT,
                 ):
                     if isinstance(internal_event, AgentBackendStreamInternalEvent):
+                        try:
+                            process_recorder.handle_stream_event(internal_event)
+                        except Exception:
+                            db.session.rollback()
+                            logger.warning(
+                                "Failed to persist Agent App process event: run_id=%s message_id=%s event_kind=%s",
+                                run_id,
+                                message_id,
+                                internal_event.event_kind,
+                                exc_info=True,
+                            )
                         text_delta = self._extract_stream_text_delta(internal_event)
                         if text_delta:
                             streamed_answer_parts.append(text_delta)

api/core/app/apps/agent_app/runtime_request_builder.py

@@ -12,7 +12,7 @@ from __future__ import annotations
 
 from collections.abc import Mapping
 from dataclasses import dataclass
-from typing import Any, Protocol, cast
+from typing import Any, Literal, Protocol, cast
 
 from agenton.compositor import CompositorSessionSnapshot
 from dify_agent.layers.execution_context import (
@@ -29,20 +29,22 @@ from clients.agent_backend import (
     redact_for_agent_backend_log,
 )
 from configs import dify_config
-from core.app.entities.app_invoke_entities import DifyRunContext
-from core.workflow.nodes.agent_v2.plugin_tools_builder import (
-    WorkflowAgentPluginToolsBuilder,
-    WorkflowAgentPluginToolsBuildError,
+from core.app.entities.app_invoke_entities import DifyRunContext, InvokeFrom
+from core.workflow.nodes.agent_v2.dify_tools_builder import (
+    WorkflowAgentDifyToolLayersBuilder,
+    WorkflowAgentDifyToolsBuilder,
+    WorkflowAgentDifyToolsBuildError,
+    WorkflowAgentToolLayers,
 )
 from core.workflow.nodes.agent_v2.runtime_request_builder import (
     append_runtime_warnings,
     build_ask_human_layer_config,
-    build_drive_aware_soul_mention_resolver,
-    build_drive_layer_config,
+    build_config_aware_soul_mention_resolver,
+    build_config_layer_config,
     build_knowledge_layer_config,
     build_shell_layer_config,
 )
-from models.agent_config_entities import AgentSoulConfig
+from models.agent_config_entities import AgentSoulConfig, AgentSoulToolsConfig
 from models.provider_ids import ModelProviderID
 from services.agent.prompt_mentions import build_soul_mention_resolver, expand_prompt_mentions
 
@@ -68,6 +70,7 @@ class AgentAppRuntimeBuildContext:
     conversation_id: str
     user_query: str
     idempotency_key: str
+    agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot"
     session_snapshot: CompositorSessionSnapshot | None = None
     # ENG-638: set when resuming a chat turn after a submitted ask_human form.
     deferred_tool_results: DeferredToolResultsPayload | None = None
@@ -88,11 +91,11 @@ class AgentAppRuntimeRequestBuilder:
         *,
         credentials_provider: CredentialsProvider,
         request_builder: AgentBackendRunRequestBuilder | None = None,
-        plugin_tools_builder: WorkflowAgentPluginToolsBuilder | None = None,
+        dify_tools_builder: WorkflowAgentDifyToolLayersBuilder | None = None,
     ) -> None:
         self._credentials_provider = credentials_provider
         self._request_builder = request_builder or AgentBackendRunRequestBuilder()
-        self._plugin_tools_builder = plugin_tools_builder or WorkflowAgentPluginToolsBuilder()
+        self._dify_tools_builder = dify_tools_builder or WorkflowAgentDifyToolsBuilder()
 
     def build(self, context: AgentAppRuntimeBuildContext) -> AgentAppRuntimeRequest:
         agent_soul = context.agent_soul
@@ -105,38 +108,33 @@ class AgentAppRuntimeRequestBuilder:
         metadata = self._build_metadata(context)
         credentials = self._credentials_provider.fetch(agent_soul.model.model_provider, agent_soul.model.model)
         try:
-            tools_layer = self._plugin_tools_builder.build(
+            tool_layers = self._build_tool_layers(
                 tenant_id=context.dify_context.tenant_id,
                 app_id=context.dify_context.app_id,
                 user_id=context.dify_context.user_id,
                 tools=agent_soul.tools,
                 invoke_from=context.dify_context.invoke_from,
             )
-        except WorkflowAgentPluginToolsBuildError as error:
+        except WorkflowAgentDifyToolsBuildError as error:
             raise AgentAppRuntimeRequestBuildError(error.error_code, str(error)) from error
-        if tools_layer is not None or agent_soul.tools.cli_tools:
+        if tool_layers.plugin_tools is not None or tool_layers.core_tools is not None or agent_soul.tools.cli_tools:
             metadata["agent_tools"] = {
-                "dify_tool_count": len(tools_layer.tools) if tools_layer is not None else 0,
-                "dify_tool_names": [tool.name or tool.tool_name for tool in tools_layer.tools]
-                if tools_layer is not None
-                else [],
+                "dify_tool_count": len(tool_layers.exposed_tool_names()),
+                "dify_tool_names": tool_layers.exposed_tool_names(),
                 "cli_tool_count": len(agent_soul.tools.cli_tools),
             }
 
-        drive_config = None
+        config_layer_config = None
         soul_prompt_resolver = build_soul_mention_resolver(agent_soul)
         if dify_config.AGENT_DRIVE_MANIFEST_ENABLED:
-            drive_config, drive_warnings = build_drive_layer_config(
+            config_layer_config, config_warnings = build_config_layer_config(
                 agent_soul,
-                tenant_id=context.dify_context.tenant_id,
-                agent_id=context.agent_id,
-            )
-            append_runtime_warnings(metadata, drive_warnings)
-            soul_prompt_resolver = build_drive_aware_soul_mention_resolver(
-                agent_soul,
-                tenant_id=context.dify_context.tenant_id,
                 agent_id=context.agent_id,
+                config_version_id=context.agent_config_snapshot_id,
+                config_version_kind=context.agent_config_version_kind,
             )
+            append_runtime_warnings(metadata, config_warnings)
+            soul_prompt_resolver = build_config_aware_soul_mention_resolver(agent_soul)
         knowledge_config = build_knowledge_layer_config(agent_soul)
 
         request = self._request_builder.build_for_agent_app(
@@ -158,6 +156,7 @@ class AgentAppRuntimeRequestBuilder:
                     conversation_id=context.conversation_id,
                     agent_id=context.agent_id,
                     agent_config_version_id=context.agent_config_snapshot_id,
+                    agent_config_version_kind=context.agent_config_version_kind,
                     # Agent Files §1.3: real Dify access context + agent run mode.
                     user_from=cast(DifyExecutionContextUserFrom, context.dify_context.user_from.value),
                     invoke_from=cast(DifyExecutionContextInvokeFrom, context.dify_context.invoke_from.value),
@@ -168,9 +167,10 @@ class AgentAppRuntimeRequestBuilder:
                 agent_soul_prompt=expand_prompt_mentions(agent_soul.prompt.system_prompt, soul_prompt_resolver).strip()
                 or None,
                 user_prompt=context.user_query,
-                tools=tools_layer,
+                tools=tool_layers.plugin_tools,
+                core_tools=tool_layers.core_tools,
                 knowledge=knowledge_config,
-                drive_config=drive_config,
+                config_layer_config=config_layer_config,
                 ask_human_config=build_ask_human_layer_config(agent_soul),
                 include_shell=dify_config.AGENT_SHELL_ENABLED,
                 shell_config=build_shell_layer_config(agent_soul),
@@ -183,6 +183,26 @@ class AgentAppRuntimeRequestBuilder:
         redacted = cast(dict[str, Any], redact_for_agent_backend_log(request))
         return AgentAppRuntimeRequest(request=request, redacted_request=redacted, metadata=metadata)
 
+    def _build_tool_layers(
+        self,
+        *,
+        tenant_id: str,
+        app_id: str,
+        user_id: str | None,
+        tools: AgentSoulToolsConfig,
+        invoke_from: InvokeFrom,
+    ) -> WorkflowAgentToolLayers:
+        # Production Agent App runs intentionally keep existing plugin configs
+        # on the direct `dify.plugin.tools` route. This builder emits plugin
+        # tools directly and non-plugin Dify tools through `dify.core.tools`.
+        return self._dify_tools_builder.build_layers(
+            tenant_id=tenant_id,
+            app_id=app_id,
+            user_id=user_id,
+            tools=tools,
+            invoke_from=invoke_from,
+        )
+
     @staticmethod
     def _build_metadata(context: AgentAppRuntimeBuildContext) -> dict[str, Any]:
         return {

api/core/app/apps/pipeline/pipeline_generator.py

@@ -144,7 +144,7 @@ class PipelineGenerator(BaseAppGenerator):
             DocumentService.check_document_creation_limits(len(datasource_info_list), features)
 
             for datasource_info in datasource_info_list:
-                position = DocumentService.get_documents_position(dataset.id)
+                position = DocumentService.get_documents_position(dataset.id, session)
                 document = self._build_document(
                     tenant_id=pipeline.tenant_id,
                     dataset_id=dataset.id,

api/core/app/entities/app_invoke_entities.py

@@ -1,8 +1,8 @@
 from collections.abc import Mapping, Sequence
 from enum import StrEnum
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Literal
 
-from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, field_validator
+from pydantic import BaseModel, ConfigDict, Field, JsonValue, ValidationInfo, field_validator
 
 from constants import UUID_NIL
 from core.app.app_config.entities import EasyUIBasedAppConfig, WorkflowUIBasedAppConfig
@@ -220,11 +220,24 @@ class AgentAppGenerateEntity(ChatAppGenerateEntity):
     accepted-entity union. The answer is produced by the dify-agent backend
     rather than an in-process LLM call; ``model_conf`` is synthesized from the
     bound Agent Soul model so the chat task pipeline can persist usage.
+
+    ``agent_config_version_kind`` selects which Agent config surface the
+    backend should read from: immutable snapshot, shared draft, or per-user
+    build draft.
+
+    ``agent_runtime_session_snapshot_id`` carries the runtime session scope
+    used to resume or suspend within the same editable config surface.
+
+    ``prompt_file_mappings`` preserves the raw request ``files`` array for the
+    Agent backend prompt. These references are appended to the backend prompt
+    text while the stored chat message keeps the user's original query.
     """
 
     agent_id: str
     agent_config_snapshot_id: str
+    agent_config_version_kind: Literal["snapshot", "draft", "build_draft"] = "snapshot"
     agent_runtime_session_snapshot_id: str | None = None
+    prompt_file_mappings: Sequence[JsonValue] = Field(default_factory=list)
 
 
 class AdvancedChatAppGenerateEntity(ConversationAppGenerateEntity):

api/core/app/features/annotation_reply/annotation_reply.py

@@ -45,7 +45,7 @@ class AnnotationReplyFeature:
             embedding_model_name = collection_binding_detail.model_name
 
             dataset_collection_binding = DatasetCollectionBindingService.get_dataset_collection_binding(
-                embedding_provider_name, embedding_model_name, CollectionBindingType.ANNOTATION
+                embedding_provider_name, embedding_model_name, db.session, CollectionBindingType.ANNOTATION
             )
 
             dataset = Dataset(

api/core/callback_handler/index_tool_callback_handler.py

@@ -2,6 +2,7 @@ import logging
 from collections.abc import Sequence
 
 from sqlalchemy import select, update
+from sqlalchemy.orm import scoped_session
 
 from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
 from core.app.entities.app_invoke_entities import InvokeFrom
@@ -9,7 +10,6 @@ from core.app.entities.queue_entities import QueueRetrieverResourcesEvent
 from core.rag.entities import RetrievalSourceMetadata
 from core.rag.index_processor.constant.index_type import IndexStructureType
 from core.rag.models.document import Document
-from extensions.ext_database import db
 from models.dataset import ChildChunk, DatasetQuery, DocumentSegment
 from models.dataset import Document as DatasetDocument
 from models.enums import CreatorUserRole, DatasetQuerySource
@@ -29,7 +29,7 @@ class DatasetIndexToolCallbackHandler:
         self._user_id = user_id
         self._invoke_from = invoke_from
 
-    def on_query(self, query: str, dataset_id: str):
+    def on_query(self, query: str, dataset_id: str, session: scoped_session):
         """
         Handle query.
         """
@@ -46,16 +46,16 @@ class DatasetIndexToolCallbackHandler:
             created_by=self._user_id,
         )
 
-        db.session.add(dataset_query)
-        db.session.commit()
+        session.add(dataset_query)
+        session.commit()
 
-    def on_tool_end(self, documents: list[Document]):
+    def on_tool_end(self, documents: list[Document], session: scoped_session):
         """Handle tool end."""
         for document in documents:
             if document.metadata is not None:
                 document_id = document.metadata["document_id"]
                 dataset_document_stmt = select(DatasetDocument).where(DatasetDocument.id == document_id)
-                dataset_document = db.session.scalar(dataset_document_stmt)
+                dataset_document = session.scalar(dataset_document_stmt)
                 if not dataset_document:
                     _logger.warning(
                         "Expected DatasetDocument record to exist, but none was found, document_id=%s",
@@ -68,9 +68,9 @@ class DatasetIndexToolCallbackHandler:
                         ChildChunk.dataset_id == dataset_document.dataset_id,
                         ChildChunk.document_id == dataset_document.id,
                     )
-                    child_chunk = db.session.scalar(child_chunk_stmt)
+                    child_chunk = session.scalar(child_chunk_stmt)
                     if child_chunk:
-                        db.session.execute(
+                        session.execute(
                             update(DocumentSegment)
                             .where(DocumentSegment.id == child_chunk.segment_id)
                             .values(hit_count=DocumentSegment.hit_count + 1)
@@ -82,11 +82,11 @@ class DatasetIndexToolCallbackHandler:
                         conditions.append(DocumentSegment.dataset_id == document.metadata["dataset_id"])
 
                     # add hit count to document segment
-                    db.session.execute(
+                    session.execute(
                         update(DocumentSegment).where(*conditions).values(hit_count=DocumentSegment.hit_count + 1)
                     )
 
-                db.session.commit()
+                session.commit()
 
     # TODO(-LAN-): Improve type check
     def return_retriever_resource_info(self, resource: Sequence[RetrievalSourceMetadata]):

api/core/callback_handler/workflow_tool_callback_handler.py

@@ -1,6 +1,7 @@
 from collections.abc import Generator, Iterable, Mapping
 from typing import Any
 
+from configs import dify_config
 from core.callback_handler.agent_tool_callback_handler import DifyAgentCallbackHandler, print_text
 from core.ops.ops_trace_manager import TraceQueueManager
 from core.tools.entities.tool_entities import ToolInvokeMessage
@@ -19,8 +20,9 @@ class DifyWorkflowCallbackHandler(DifyAgentCallbackHandler):
         trace_manager: TraceQueueManager | None = None,
     ) -> Generator[ToolInvokeMessage, None, None]:
         for tool_output in tool_outputs:
-            print_text("\n[on_tool_execution]\n", color=self.color)
-            print_text("Tool: " + tool_name + "\n", color=self.color)
-            print_text("Outputs: " + tool_output.model_dump_json()[:1000] + "\n", color=self.color)
-            print_text("\n")
+            if dify_config.DEBUG:
+                print_text("\n[on_tool_execution]\n", color=self.color)
+                print_text("Tool: " + tool_name + "\n", color=self.color)
+                print_text("Outputs: " + tool_output.model_dump_json()[:1000] + "\n", color=self.color)
+                print_text("\n")
             yield tool_output

api/core/entities/provider_configuration.py

@@ -22,7 +22,10 @@ from core.entities.provider_entities import (
     SystemConfigurationStatus,
 )
 from core.helper import encrypter
-from core.helper.model_provider_cache import ProviderCredentialsCache, ProviderCredentialsCacheType
+from core.helper.model_provider_cache import (
+    ProviderCredentialsCache,
+    ProviderCredentialsCacheType,
+)
 from core.plugin.impl.model_runtime_factory import create_model_type_instance, create_plugin_model_assembly
 from graphon.model_runtime.entities.model_entities import AIModelEntity, FetchFrom, ModelType
 from graphon.model_runtime.entities.provider_entities import (
@@ -473,6 +476,39 @@ class ProviderConfiguration(BaseModel):
             provider_names.append(model_provider_id.provider_name)
         return provider_names
 
+    def _invalidate_provider_configuration_cache(
+        self,
+        *,
+        provider_models: bool = False,
+        preferred_model_providers: bool = False,
+        provider_model_settings: bool = False,
+        provider_model_credentials: bool = False,
+        provider_credentials: bool = False,
+        provider_load_balancing_configs: bool = False,
+    ) -> None:
+        """Invalidate tenant-scoped provider snapshots after committing configuration writes."""
+        from core.provider_manager import ProviderConfigurationCacheSource, ProviderManager
+
+        sources: list[ProviderConfigurationCacheSource] = []
+        if provider_models:
+            sources.append(ProviderConfigurationCacheSource.PROVIDER_MODELS)
+        if preferred_model_providers:
+            sources.append(ProviderConfigurationCacheSource.PREFERRED_MODEL_PROVIDERS)
+        if provider_model_settings:
+            sources.append(ProviderConfigurationCacheSource.PROVIDER_MODEL_SETTINGS)
+        if provider_model_credentials:
+            sources.append(ProviderConfigurationCacheSource.PROVIDER_MODEL_CREDENTIALS)
+        if provider_credentials:
+            sources.append(ProviderConfigurationCacheSource.PROVIDER_CREDENTIALS)
+        if provider_load_balancing_configs:
+            sources.append(ProviderConfigurationCacheSource.PROVIDER_LOAD_BALANCING_CONFIGS)
+
+        if not sources:
+            logger.warning("No provider configuration cache source selected for invalidation")
+            return
+
+        ProviderManager.invalidate_configurations_cache(self.tenant_id, sources=sources)
+
     def create_provider_credential(self, credentials: dict[str, Any], credential_name: str | None):
         """
         Add custom provider credentials.
@@ -489,6 +525,7 @@ class ProviderConfiguration(BaseModel):
 
         credentials = self.validate_provider_credentials(credentials=credentials)
 
+        preferred_model_providers_changed = False
         with Session(db.engine) as session:
             provider_record = self._get_provider_record(session)
             try:
@@ -518,7 +555,9 @@ class ProviderConfiguration(BaseModel):
                     )
                     provider_model_credentials_cache.delete()
 
-                    self.switch_preferred_provider_type(provider_type=ProviderType.CUSTOM, session=session)
+                    preferred_model_providers_changed = self.switch_preferred_provider_type(
+                        provider_type=ProviderType.CUSTOM, session=session
+                    )
                 else:
                     provider_record.is_valid = True
 
@@ -533,12 +572,18 @@ class ProviderConfiguration(BaseModel):
                         )
                         provider_model_credentials_cache.delete()
 
-                        self.switch_preferred_provider_type(provider_type=ProviderType.CUSTOM, session=session)
+                        preferred_model_providers_changed = self.switch_preferred_provider_type(
+                            provider_type=ProviderType.CUSTOM, session=session
+                        )
 
                 session.commit()
             except Exception:
                 session.rollback()
                 raise
+        self._invalidate_provider_configuration_cache(
+            preferred_model_providers=preferred_model_providers_changed,
+            provider_credentials=True,
+        )
 
     def update_provider_credential(
         self,
@@ -562,6 +607,7 @@ class ProviderConfiguration(BaseModel):
 
         credentials = self.validate_provider_credentials(credentials=credentials, credential_id=credential_id)
 
+        load_balancing_configs_changed = False
         with Session(db.engine) as session:
             provider_record = self._get_provider_record(session)
             stmt = select(ProviderCredential).where(
@@ -588,7 +634,7 @@ class ProviderConfiguration(BaseModel):
                     )
                     provider_model_credentials_cache.delete()
 
-                self._update_load_balancing_configs_with_credential(
+                load_balancing_configs_changed = self._update_load_balancing_configs_with_credential(
                     credential_id=credential_id,
                     credential_record=credential_record,
                     credential_source=CredentialSourceType.PROVIDER,
@@ -597,6 +643,10 @@ class ProviderConfiguration(BaseModel):
             except Exception:
                 session.rollback()
                 raise
+        self._invalidate_provider_configuration_cache(
+            provider_credentials=True,
+            provider_load_balancing_configs=load_balancing_configs_changed,
+        )
 
     def _update_load_balancing_configs_with_credential(
         self,
@@ -604,7 +654,7 @@ class ProviderConfiguration(BaseModel):
         credential_record: ProviderCredential | ProviderModelCredential,
         credential_source: str,
         session: Session,
-    ):
+    ) -> bool:
         """
         Update load balancing configurations that reference the given credential_id.
 
@@ -625,7 +675,7 @@ class ProviderConfiguration(BaseModel):
         load_balancing_configs = session.execute(stmt).scalars().all()
 
         if not load_balancing_configs:
-            return
+            return False
 
         # Update each load balancing config with the new credentials
         for lb_config in load_balancing_configs:
@@ -643,6 +693,7 @@ class ProviderConfiguration(BaseModel):
             lb_credentials_cache.delete()
 
         session.commit()
+        return True
 
     def delete_provider_credential(self, credential_id: str):
         """
@@ -651,6 +702,8 @@ class ProviderConfiguration(BaseModel):
         :param credential_id: credential id
         :return:
         """
+        preferred_model_providers_changed = False
+        load_balancing_configs_changed = False
         with Session(db.engine) as session:
             stmt = select(ProviderCredential).where(
                 ProviderCredential.id == credential_id,
@@ -671,6 +724,7 @@ class ProviderConfiguration(BaseModel):
                 LoadBalancingModelConfig.credential_source_type == CredentialSourceType.PROVIDER,
             )
             lb_configs_using_credential = session.execute(lb_stmt).scalars().all()
+            load_balancing_configs_changed = bool(lb_configs_using_credential)
             try:
                 for lb_config in lb_configs_using_credential:
                     lb_credentials_cache = ProviderCredentialsCache(
@@ -703,7 +757,9 @@ class ProviderConfiguration(BaseModel):
                         cache_type=ProviderCredentialsCacheType.PROVIDER,
                     )
                     provider_model_credentials_cache.delete()
-                    self.switch_preferred_provider_type(provider_type=ProviderType.SYSTEM, session=session)
+                    preferred_model_providers_changed = self.switch_preferred_provider_type(
+                        provider_type=ProviderType.SYSTEM, session=session
+                    )
                 elif provider_record and provider_record.credential_id == credential_id:
                     provider_record.credential_id = None
                     provider_record.updated_at = naive_utc_now()
@@ -714,12 +770,19 @@ class ProviderConfiguration(BaseModel):
                         cache_type=ProviderCredentialsCacheType.PROVIDER,
                     )
                     provider_model_credentials_cache.delete()
-                    self.switch_preferred_provider_type(provider_type=ProviderType.SYSTEM, session=session)
+                    preferred_model_providers_changed = self.switch_preferred_provider_type(
+                        provider_type=ProviderType.SYSTEM, session=session
+                    )
 
                 session.commit()
             except Exception:
                 session.rollback()
                 raise
+        self._invalidate_provider_configuration_cache(
+            preferred_model_providers=preferred_model_providers_changed,
+            provider_credentials=True,
+            provider_load_balancing_configs=load_balancing_configs_changed,
+        )
 
     def switch_active_provider_credential(self, credential_id: str):
         """
@@ -728,6 +791,7 @@ class ProviderConfiguration(BaseModel):
         :param credential_id: credential id
         :return:
         """
+        preferred_model_providers_changed = False
         with Session(db.engine) as session:
             stmt = select(ProviderCredential).where(
                 ProviderCredential.id == credential_id,
@@ -753,10 +817,14 @@ class ProviderConfiguration(BaseModel):
                     cache_type=ProviderCredentialsCacheType.PROVIDER,
                 )
                 provider_model_credentials_cache.delete()
-                self.switch_preferred_provider_type(ProviderType.CUSTOM, session=session)
+                preferred_model_providers_changed = self.switch_preferred_provider_type(
+                    ProviderType.CUSTOM, session=session
+                )
             except Exception:
                 session.rollback()
                 raise
+        if preferred_model_providers_changed:
+            self._invalidate_provider_configuration_cache(preferred_model_providers=True)
 
     def _get_custom_model_record(
         self,
@@ -1017,6 +1085,10 @@ class ProviderConfiguration(BaseModel):
             except Exception:
                 session.rollback()
                 raise
+        self._invalidate_provider_configuration_cache(
+            provider_models=True,
+            provider_model_credentials=True,
+        )
 
     def update_custom_model_credential(
         self,
@@ -1053,6 +1125,7 @@ class ProviderConfiguration(BaseModel):
             credential_id=credential_id,
         )
 
+        load_balancing_configs_changed = False
         with Session(db.engine) as session:
             provider_model_record = self._get_custom_model_record(model_type=model_type, model=model, session=session)
 
@@ -1082,7 +1155,7 @@ class ProviderConfiguration(BaseModel):
                     )
                     provider_model_credentials_cache.delete()
 
-                self._update_load_balancing_configs_with_credential(
+                load_balancing_configs_changed = self._update_load_balancing_configs_with_credential(
                     credential_id=credential_id,
                     credential_record=credential_record,
                     credential_source=CredentialSourceType.CUSTOM_MODEL,
@@ -1091,6 +1164,11 @@ class ProviderConfiguration(BaseModel):
             except Exception:
                 session.rollback()
                 raise
+        self._invalidate_provider_configuration_cache(
+            provider_models=True,
+            provider_model_credentials=True,
+            provider_load_balancing_configs=load_balancing_configs_changed,
+        )
 
     def delete_custom_model_credential(self, model_type: ModelType, model: str, credential_id: str):
         """
@@ -1099,6 +1177,7 @@ class ProviderConfiguration(BaseModel):
         :param credential_id: credential id
         :return:
         """
+        load_balancing_configs_changed = False
         with Session(db.engine) as session:
             stmt = select(ProviderModelCredential).where(
                 ProviderModelCredential.id == credential_id,
@@ -1118,6 +1197,7 @@ class ProviderConfiguration(BaseModel):
                 LoadBalancingModelConfig.credential_source_type == CredentialSourceType.CUSTOM_MODEL,
             )
             lb_configs_using_credential = session.execute(lb_stmt).scalars().all()
+            load_balancing_configs_changed = bool(lb_configs_using_credential)
 
             try:
                 for lb_config in lb_configs_using_credential:
@@ -1161,6 +1241,11 @@ class ProviderConfiguration(BaseModel):
             except Exception:
                 session.rollback()
                 raise
+        self._invalidate_provider_configuration_cache(
+            provider_models=True,
+            provider_model_credentials=True,
+            provider_load_balancing_configs=load_balancing_configs_changed,
+        )
 
     def add_model_credential_to_model(self, model_type: ModelType, model: str, credential_id: str):
         """
@@ -1213,6 +1298,7 @@ class ProviderConfiguration(BaseModel):
 
             session.add(provider_model_record)
             session.commit()
+        self._invalidate_provider_configuration_cache(provider_models=True)
 
     def switch_custom_model_credential(self, model_type: ModelType, model: str, credential_id: str):
         """
@@ -1251,6 +1337,7 @@ class ProviderConfiguration(BaseModel):
                 cache_type=ProviderCredentialsCacheType.MODEL,
             )
             provider_model_credentials_cache.delete()
+        self._invalidate_provider_configuration_cache(provider_models=True)
 
     def delete_custom_model(self, model_type: ModelType, model: str):
         """
@@ -1259,6 +1346,7 @@ class ProviderConfiguration(BaseModel):
         :param model: model name
         :return:
         """
+        provider_models_changed = False
         with Session(db.engine) as session:
             # get provider model
             provider_model_record = self._get_custom_model_record(model_type=model_type, model=model, session=session)
@@ -1267,6 +1355,7 @@ class ProviderConfiguration(BaseModel):
             if provider_model_record:
                 session.delete(provider_model_record)
                 session.commit()
+                provider_models_changed = True
 
                 provider_model_credentials_cache = ProviderCredentialsCache(
                     tenant_id=self.tenant_id,
@@ -1275,6 +1364,8 @@ class ProviderConfiguration(BaseModel):
                 )
 
                 provider_model_credentials_cache.delete()
+        if provider_models_changed:
+            self._invalidate_provider_configuration_cache(provider_models=True)
 
     def _get_provider_model_setting(
         self, model_type: ModelType, model: str, session: Session
@@ -1314,6 +1405,7 @@ class ProviderConfiguration(BaseModel):
                 )
                 session.add(model_setting)
             session.commit()
+        self._invalidate_provider_configuration_cache(provider_model_settings=True)
 
         return model_setting
 
@@ -1340,6 +1432,7 @@ class ProviderConfiguration(BaseModel):
                 )
                 session.add(model_setting)
             session.commit()
+        self._invalidate_provider_configuration_cache(provider_model_settings=True)
 
         return model_setting
 
@@ -1392,6 +1485,7 @@ class ProviderConfiguration(BaseModel):
                 )
                 session.add(model_setting)
             session.commit()
+        self._invalidate_provider_configuration_cache(provider_model_settings=True)
 
         return model_setting
 
@@ -1419,6 +1513,7 @@ class ProviderConfiguration(BaseModel):
                 )
                 session.add(model_setting)
             session.commit()
+        self._invalidate_provider_configuration_cache(provider_model_settings=True)
 
         return model_setting
 
@@ -1454,19 +1549,19 @@ class ProviderConfiguration(BaseModel):
             credentials=credentials or {},
         )
 
-    def switch_preferred_provider_type(self, provider_type: ProviderType, session: Session | None = None):
+    def switch_preferred_provider_type(self, provider_type: ProviderType, session: Session | None = None) -> bool:
         """
         Switch preferred provider type.
         :param provider_type:
         :return:
         """
         if provider_type == self.preferred_provider_type:
-            return
+            return False
 
         if provider_type == ProviderType.SYSTEM and not self.system_configuration.enabled:
-            return
+            return False
 
-        def _switch(s: Session):
+        def _switch(s: Session) -> bool:
             stmt = select(TenantPreferredModelProvider).where(
                 TenantPreferredModelProvider.tenant_id == self.tenant_id,
                 TenantPreferredModelProvider.provider_name.in_(self._get_provider_names()),
@@ -1483,12 +1578,16 @@ class ProviderConfiguration(BaseModel):
                 )
                 s.add(preferred_model_provider)
             s.commit()
+            return True
 
         if session:
             return _switch(session)
         else:
             with Session(db.engine) as session:
-                return _switch(session)
+                changed = _switch(session)
+            if changed:
+                self._invalidate_provider_configuration_cache(preferred_model_providers=True)
+            return changed
 
     def extract_secret_variables(self, credential_form_schemas: list[CredentialFormSchema]) -> list[str]:
         """

api/core/llm_generator/llm_generator.py

@@ -2,7 +2,7 @@ import json
 import logging
 import re
 from collections.abc import Sequence
-from typing import Any, NotRequired, Protocol, TypedDict, cast
+from typing import Any, Literal, NotRequired, Protocol, TypedDict, cast
 
 import json_repair
 from sqlalchemy import select
@@ -69,6 +69,53 @@ def _normalize_completion_params(completion_params: dict[str, object]) -> tuple[
     return normalized_parameters, stop
 
 
+# ── Workflow instruction-suggestion tuning ────────────────────────────────
+# Suggestions are a soft, pre-model-pick enhancement: short, buildable example
+# instructions proposed from the tenant's DEFAULT model. Every failure path
+# degrades to an empty list, never an error.
+_SUGGESTION_MIN_COUNT = 1
+_SUGGESTION_MAX_COUNT = 6
+_SUGGESTION_MAX_TOKENS = 512
+_SUGGESTION_TEMPERATURE = 0.8
+# Bound the grounding context so the prompt stays small regardless of how many
+# knowledge bases / tools the tenant has installed.
+_SUGGESTION_KB_LIMIT = 10
+_SUGGESTION_TOOL_SAMPLE_LINES = 20
+
+_SUGGESTION_SYSTEM_PROMPT = (
+    "You help a user start building a Dify app by proposing example build instructions. "
+    "Each suggestion must be a SHORT (at most 8 words), concrete, and BUILDABLE instruction "
+    "describing an app to generate for the given app type. Make the suggestions diverse — cover "
+    "different use cases. When the listed knowledge bases or installed tools fit a suggestion, "
+    "prefer them, but NEVER invent tools or knowledge bases that are not listed. "
+    "Reply with ONLY a JSON array of strings and nothing else."
+)
+
+
+def _parse_string_list(text: str) -> list[str]:
+    """Extract a JSON array of strings from a (possibly noisy) LLM response.
+
+    Slices the first ``[...]`` span so surrounding prose / markdown fences are
+    tolerated, parses it with ``json`` and falls back to ``json_repair``, then
+    keeps only ``str`` items. Returns ``[]`` on any failure so callers can
+    treat parsing as best-effort.
+    """
+    match = re.search(r"\[.*\]", text.strip(), re.DOTALL)
+    if not match:
+        return []
+    raw = match.group(0)
+    try:
+        parsed = json.loads(raw)
+    except Exception:
+        try:
+            parsed = json_repair.loads(raw)
+        except Exception:
+            return []
+    if not isinstance(parsed, list):
+        return []
+    return [item for item in parsed if isinstance(item, str)]
+
+
 class WorkflowServiceInterface(Protocol):
     def get_draft_workflow(self, app_model: App, workflow_id: str | None = None) -> Workflow | None:
         pass
@@ -237,6 +284,170 @@ class LLMGenerator:
 
         return questions
 
+    @classmethod
+    def generate_workflow_instruction_suggestions(
+        cls,
+        tenant_id: str,
+        *,
+        mode: Literal["workflow", "advanced-chat"],
+        language: str | None = None,
+        count: int = 4,
+    ) -> list[str]:
+        """Propose short, buildable example instructions for the workflow generator.
+
+        Runs BEFORE the user picks a model, so it uses the tenant's DEFAULT LLM
+        only. Suggestions are a soft enhancement, never a blocker: every failure
+        path (no default model, KB / tool lookup error, LLM error, unparseable
+        output) is swallowed and surfaced as an empty list — a valid result the
+        caller renders as "no suggestions". This method NEVER raises.
+        """
+        count = max(_SUGGESTION_MIN_COUNT, min(count, _SUGGESTION_MAX_COUNT))
+
+        try:
+            model_instance = ModelManager.for_tenant(tenant_id=tenant_id).get_default_model_instance(
+                tenant_id=tenant_id,
+                model_type=ModelType.LLM,
+            )
+        except Exception:
+            logger.info("Workflow instruction suggestions: no default model for tenant %s", tenant_id)
+            return []
+
+        context_block = cls._build_suggestion_context(tenant_id)
+        app_type_label = (
+            "Workflow — single-shot automation" if mode == "workflow" else "Chatflow — conversational multi-turn"
+        )
+
+        user_lines = [
+            f"App type: {app_type_label}",
+            context_block,
+            f"Return exactly {count} distinct ideas as a JSON array of strings.",
+        ]
+        if language:
+            user_lines.append(f"Write every idea in this language: {language}.")
+        user_prompt = "\n".join(line for line in user_lines if line)
+
+        prompt_messages: list[PromptMessage] = [
+            SystemPromptMessage(content=_SUGGESTION_SYSTEM_PROMPT),
+            UserPromptMessage(content=user_prompt),
+        ]
+
+        try:
+            response: LLMResult = model_instance.invoke_llm(
+                prompt_messages=prompt_messages,
+                model_parameters={"max_tokens": _SUGGESTION_MAX_TOKENS, "temperature": _SUGGESTION_TEMPERATURE},
+                stream=False,
+            )
+        except Exception:
+            logger.exception("Workflow instruction suggestions: LLM invocation failed")
+            return []
+
+        raw_suggestions = _parse_string_list(response.message.get_text_content() or "")
+
+        # Strip whitespace + surrounding quotes, drop empties, dedupe
+        # case-insensitively (preserving first-seen casing), cap to ``count``.
+        cleaned: list[str] = []
+        seen: set[str] = set()
+        for item in raw_suggestions:
+            idea = item.strip().strip("\"'").strip()
+            if not idea:
+                continue
+            key = idea.casefold()
+            if key in seen:
+                continue
+            seen.add(key)
+            cleaned.append(idea)
+            if len(cleaned) >= count:
+                break
+        return cleaned
+
+    @staticmethod
+    def _build_suggestion_context(tenant_id: str) -> str:
+        """Assemble an optional grounding block naming the tenant's KBs and tools.
+
+        Best-effort: each section is isolated in its own try/except so a failure
+        enumerating one (DB hiccup, plugin daemon down) never blocks the other
+        or the suggestion call itself. Returns "" when nothing is available.
+        """
+        sections: list[str] = []
+
+        try:
+            from models.dataset import Dataset
+
+            names = db.session.scalars(
+                select(Dataset.name)
+                .where(Dataset.tenant_id == tenant_id)
+                .order_by(Dataset.created_at.desc())
+                .limit(_SUGGESTION_KB_LIMIT)
+            ).all()
+            kb_names = [name for name in names if name]
+            if kb_names:
+                sections.append("Knowledge bases:\n" + "\n".join(f"- {name}" for name in kb_names))
+        except Exception:
+            logger.info("Workflow instruction suggestions: failed to load knowledge bases", exc_info=True)
+
+        try:
+            from core.workflow.generator.tool_catalogue import build_tool_catalogue, format_tool_catalogue
+
+            tool_text = format_tool_catalogue(build_tool_catalogue(tenant_id))
+            if tool_text:
+                sample = "\n".join(tool_text.splitlines()[:_SUGGESTION_TOOL_SAMPLE_LINES])
+                sections.append("Installed tools:\n" + sample)
+        except Exception:
+            logger.info("Workflow instruction suggestions: failed to load tool catalogue", exc_info=True)
+
+        if not sections:
+            return ""
+        return "\n\n".join(sections) + "\n\n"
+
+    @classmethod
+    def classify_workflow_mode(
+        cls,
+        tenant_id: str,
+        instruction: str,
+        model_config: ModelConfig,
+    ) -> Literal["workflow", "advanced-chat"]:
+        """Classify a free-text instruction into a concrete app mode.
+
+        One tiny LLM call using the model the user already picked (so no extra
+        provider setup is needed). Parsed leniently; defaults to
+        ``advanced-chat`` on anything unexpected or any error, so a
+        ``mode="auto"`` request never blocks generation. NEVER raises.
+        """
+        default_mode: Literal["workflow", "advanced-chat"] = "advanced-chat"
+        try:
+            model_instance = ModelManager.for_tenant(tenant_id=tenant_id).get_model_instance(
+                tenant_id=tenant_id,
+                model_type=ModelType.LLM,
+                provider=model_config.provider,
+                model=model_config.name,
+            )
+            prompt_messages: list[PromptMessage] = [
+                UserPromptMessage(
+                    content=(
+                        "Reply with exactly one word: 'workflow' (one-shot automation, no chat) "
+                        "or 'advanced-chat' (conversational multi-turn). "
+                        f"Instruction: {instruction.strip()}"
+                    )
+                ),
+            ]
+            response: LLMResult = model_instance.invoke_llm(
+                prompt_messages=prompt_messages,
+                model_parameters={"max_tokens": 4, "temperature": 0},
+                stream=False,
+            )
+            text = (response.message.get_text_content() or "").strip().lower()
+        except Exception:
+            logger.info("Workflow mode classification failed; defaulting to %s", default_mode, exc_info=True)
+            return default_mode
+
+        # Lenient parse: an affirmative "workflow" wins; everything else
+        # (including a truncated / empty / garbled reply) falls back to the
+        # conversational default. "advanced-chat" needs no positive match
+        # because it IS the default.
+        if "workflow" in text:
+            return "workflow"
+        return default_mode
+
     @classmethod
     def generate_rule_config(cls, tenant_id: str, args: RuleGeneratePayload):
         output_parser = RuleConfigGeneratorOutputParser()
@@ -498,7 +709,11 @@ class LLMGenerator:
         ideal_output: str | None,
     ):
         last_run: Message | None = db.session.scalar(
-            select(Message).where(Message.app_id == flow_id).order_by(Message.created_at.desc()).limit(1)
+            select(Message)
+            .join(App, App.id == Message.app_id)
+            .where(Message.app_id == flow_id, App.tenant_id == tenant_id)
+            .order_by(Message.created_at.desc())
+            .limit(1)
         )
         if not last_run:
             return LLMGenerator.__instruction_modify_common(
@@ -540,7 +755,7 @@ class LLMGenerator:
     ):
         session = db.session()
 
-        app: App | None = session.scalar(select(App).where(App.id == flow_id).limit(1))
+        app: App | None = session.scalar(select(App).where(App.id == flow_id, App.tenant_id == tenant_id).limit(1))
         if not app:
             raise ValueError("App not found.")
         workflow = workflow_service.get_draft_workflow(app_model=app)

api/core/mcp/mcp_client.py

@@ -1,10 +1,13 @@
 import logging
+import re
 from collections.abc import Callable
 from contextlib import AbstractContextManager, ExitStack
 from types import TracebackType
 from typing import Any
 from urllib.parse import urlparse
 
+from flask import has_request_context, request
+
 from core.mcp.client.sse_client import sse_client
 from core.mcp.client.streamable_client import streamablehttp_client
 from core.mcp.error import MCPConnectionError
@@ -23,10 +26,22 @@ class MCPClient:
         sse_read_timeout: float | None = None,
     ):
         self.server_url = server_url
-        self.headers = headers or {}
+        self.headers = headers.copy() if headers else {}
         self.timeout = timeout
         self.sse_read_timeout = sse_read_timeout
 
+        # Substitute placeholders with incoming request headers if in a request context
+        if has_request_context() and self.headers:
+            pattern = re.compile(r"\{\{\s*request\.headers?\.(.+?)\s*\}\}", re.IGNORECASE)
+            for key, value in list(self.headers.items()):
+                if isinstance(value, str):
+
+                    def replace_func(match):
+                        header_name = match.group(1)
+                        return request.headers.get(header_name, "")
+
+                    self.headers[key] = pattern.sub(replace_func, value)
+
         # Initialize session and client objects
         self._session: ClientSession | None = None
         self._exit_stack = ExitStack()

api/core/plugin/entities/marketplace.py

@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, Literal
 
 from pydantic import BaseModel, Field, computed_field, model_validator
 
@@ -32,7 +32,9 @@ class MarketplacePluginDeclaration(BaseModel):
     latest_package_identifier: str = Field(
         ..., description="Unique identifier for the latest package release of the plugin"
     )
-    status: str = Field(..., description="Indicate the status of marketplace plugin, enum from `active` `deleted`")
+    status: Literal["active", "deleted"] = Field(
+        ..., description="Indicate the status of marketplace plugin, enum from `active` `deleted`"
+    )
     deprecated_reason: str = Field(
         ..., description="Not empty when status='deleted', indicates the reason why this plugin is deleted(deprecated)"
     )

api/core/plugin/plugin_service.py

@@ -16,10 +16,11 @@ import logging
 import time
 from collections.abc import Mapping, Sequence
 from mimetypes import guess_type
-from typing import ClassVar
+from typing import Any, ClassVar, Literal
 
 from pydantic import BaseModel, TypeAdapter, ValidationError
 from redis import RedisError
+from redis.exceptions import LockError
 from sqlalchemy import delete, select, update
 from sqlalchemy.orm import Session
 from yarl import URL
@@ -74,7 +75,7 @@ class PluginService:
         plugin_id: str
         version: str
         unique_identifier: str
-        status: str
+        status: Literal["active", "deleted"]
         deprecated_reason: str
         alternative_plugin_id: str
 
@@ -82,6 +83,10 @@ class PluginService:
     REDIS_TTL = 60 * 5  # 5 minutes
     PLUGIN_MODEL_PROVIDERS_REDIS_KEY_PREFIX = "plugin_model_providers:tenant_id:"
     PLUGIN_MODEL_PROVIDERS_GENERATION_REDIS_KEY_PREFIX = "plugin_model_providers_generation:tenant_id:"
+    PLUGIN_MODEL_PROVIDERS_LOCK_REDIS_KEY_PREFIX = "plugin_model_providers_refresh_lock:tenant_id:"
+    PLUGIN_MODEL_PROVIDERS_LOCK_TTL = 30
+    PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_TIMEOUT = 2.0
+    PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_INTERVAL = 0.05
     PLUGIN_INSTALL_TASK_TERMINAL_STATUSES = (PluginInstallTaskStatus.Success, PluginInstallTaskStatus.Failed)
     # Mirror the detail-panel endpoint query size so list reconciliation and
     # the visible endpoint drawer exercise the same daemon pagination path.
@@ -98,6 +103,10 @@ class PluginService:
     def _get_plugin_model_providers_generation_cache_key(cls, tenant_id: str) -> str:
         return f"{cls.PLUGIN_MODEL_PROVIDERS_GENERATION_REDIS_KEY_PREFIX}{tenant_id}"
 
+    @classmethod
+    def _get_plugin_model_providers_lock_key(cls, tenant_id: str, generation: int) -> str:
+        return f"{cls.PLUGIN_MODEL_PROVIDERS_LOCK_REDIS_KEY_PREFIX}{tenant_id}:generation:{generation}"
+
     @staticmethod
     def _get_provider_short_name_alias(provider: PluginModelProviderEntity) -> str:
         """
@@ -197,32 +206,41 @@ class PluginService:
         cls, tenant_id: str, *, client: PluginModelClient | None = None
     ) -> tuple[ProviderEntity, ...] | None:
         generation = cls._load_plugin_model_providers_generation(tenant_id)
+        cached_providers, _ = cls._load_cached_plugin_model_providers_for_generation(tenant_id, generation)
+        return cached_providers
+
+    @classmethod
+    def _load_cached_plugin_model_providers_for_generation(
+        cls, tenant_id: str, generation: int | None
+    ) -> tuple[tuple[ProviderEntity, ...] | None, bool]:
         if generation is not None:
             in_memory_cached_providers = cls._load_in_memory_plugin_model_providers(tenant_id, generation)
             if in_memory_cached_providers is not None:
-                return in_memory_cached_providers
+                return in_memory_cached_providers, True
+
+        if generation is None:
+            return None, False
 
         cache_keys = []
-        if generation is not None:
-            cache_keys.append(cls._get_plugin_model_providers_cache_key(tenant_id, generation))
-            if generation == 0:
-                cache_keys.append(cls._get_plugin_model_providers_cache_key(tenant_id))
+        cache_keys.append(cls._get_plugin_model_providers_cache_key(tenant_id, generation))
+        if generation == 0:
+            cache_keys.append(cls._get_plugin_model_providers_cache_key(tenant_id))
 
         if not cache_keys:
-            return None
+            return None, True
 
         try:
             cached_provider_entries = redis_client.mget(cache_keys)
-        except (RedisError, RuntimeError):
+        except (LockError, RedisError, RuntimeError):
             logger.warning("Failed to read cached plugin model providers for tenant %s.", tenant_id, exc_info=True)
-            return None
+            return None, False
 
         if len(cached_provider_entries) != len(cache_keys):
             logger.warning(
                 "Unexpected cached plugin model providers response size for tenant %s.",
                 tenant_id,
             )
-            return None
+            return None, False
 
         for cache_key, cached_providers in zip(cache_keys, cached_provider_entries):
             if not cached_providers:
@@ -232,7 +250,7 @@ class PluginService:
                 providers = tuple(_provider_entities_adapter.validate_json(cached_providers))
                 if generation is not None:
                     cls._store_in_memory_plugin_model_providers(tenant_id, generation, providers)
-                return providers
+                return providers, True
             except (TypeError, ValueError, ValidationError):
                 logger.warning(
                     "Invalid cached plugin model providers for tenant %s; deleting cache key %s.",
@@ -249,7 +267,7 @@ class PluginService:
                         exc_info=True,
                     )
 
-        return None
+        return None, True
 
     @classmethod
     def _store_cached_plugin_model_providers(
@@ -262,6 +280,49 @@ class PluginService:
         except (RedisError, RuntimeError):
             logger.warning("Failed to cache plugin model providers for tenant %s.", tenant_id, exc_info=True)
 
+    @classmethod
+    def _try_acquire_plugin_model_providers_lock(cls, tenant_id: str, generation: int) -> tuple[Any | None, bool]:
+        lock_key = cls._get_plugin_model_providers_lock_key(tenant_id, generation)
+        try:
+            lock = redis_client.lock(lock_key, timeout=cls.PLUGIN_MODEL_PROVIDERS_LOCK_TTL, blocking=False)
+            acquired = lock.acquire(blocking=False)
+        except (RedisError, RuntimeError):
+            logger.warning(
+                "Failed to acquire plugin model providers refresh lock for tenant %s.",
+                tenant_id,
+                exc_info=True,
+            )
+            return None, False
+
+        if not acquired:
+            return None, True
+
+        return lock, True
+
+    @classmethod
+    def _release_plugin_model_providers_lock(cls, tenant_id: str, lock: Any) -> None:
+        try:
+            lock.release()
+        except (LockError, RedisError, RuntimeError):
+            logger.warning(
+                "Failed to release plugin model providers refresh lock for tenant %s.",
+                tenant_id,
+                exc_info=True,
+            )
+
+    @classmethod
+    def _wait_for_plugin_model_providers_refresh(
+        cls, tenant_id: str, *, client: PluginModelClient | None = None
+    ) -> tuple[ProviderEntity, ...] | None:
+        deadline = time.monotonic() + cls.PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_TIMEOUT
+        while time.monotonic() < deadline:
+            time.sleep(cls.PLUGIN_MODEL_PROVIDERS_LOCK_WAIT_INTERVAL)
+            cached_providers = cls._load_cached_plugin_model_providers(tenant_id, client=client)
+            if cached_providers is not None:
+                return cached_providers
+
+        return None
+
     @classmethod
     def invalidate_plugin_model_providers_cache(cls, tenant_id: str) -> None:
         """Invalidate tenant-scoped provider metadata across Redis and worker-local mirrors."""
@@ -287,21 +348,38 @@ class PluginService:
         are intentionally owned by this service so tenant isolation and cache
         expiry are handled in one place.
         """
-        cached_providers = cls._load_cached_plugin_model_providers(tenant_id, client=client)
+        generation = cls._load_plugin_model_providers_generation(tenant_id)
+        cached_providers, cache_available = cls._load_cached_plugin_model_providers_for_generation(
+            tenant_id, generation
+        )
         if cached_providers is not None:
             return cached_providers
 
+        refresh_lock: Any | None = None
+        refresh_generation = generation
+        if generation is not None and cache_available:
+            lock_wait_deadline = time.monotonic() + cls.PLUGIN_MODEL_PROVIDERS_LOCK_TTL
+            while time.monotonic() < lock_wait_deadline:
+                refresh_lock, lock_available = cls._try_acquire_plugin_model_providers_lock(tenant_id, generation)
+                if refresh_lock is not None or not lock_available:
+                    break
+                refreshed_providers = cls._wait_for_plugin_model_providers_refresh(tenant_id, client=client)
+                if refreshed_providers is not None:
+                    return refreshed_providers
+
         model_client = client or PluginModelClient()
-        providers = tuple(
-            cls._to_provider_entity(provider) for provider in model_client.fetch_model_providers(tenant_id)
-        )
-        if not providers:
+        try:
+            providers = tuple(
+                cls._to_provider_entity(provider) for provider in model_client.fetch_model_providers(tenant_id)
+            )
+            generation = cls._load_plugin_model_providers_generation(tenant_id)
+            if generation is not None and generation == refresh_generation:
+                cls._store_in_memory_plugin_model_providers(tenant_id, generation, providers)
+                cls._store_cached_plugin_model_providers(tenant_id, generation, providers)
             return providers
-        generation = cls._load_plugin_model_providers_generation(tenant_id)
-        if generation is not None:
-            cls._store_in_memory_plugin_model_providers(tenant_id, generation, providers)
-            cls._store_cached_plugin_model_providers(tenant_id, generation, providers)
-        return providers
+        finally:
+            if refresh_lock is not None:
+                cls._release_plugin_model_providers_lock(tenant_id, refresh_lock)
 
     @staticmethod
     def fetch_latest_plugin_version(plugin_ids: Sequence[str]) -> Mapping[str, LatestPluginCache | None]:

api/core/provider_manager.py

@@ -1,10 +1,14 @@
 from __future__ import annotations
 
 import contextlib
+import json
+import logging
 from collections import defaultdict
-from collections.abc import Sequence
+from collections.abc import Callable, Sequence
+from dataclasses import asdict, dataclass
+from enum import StrEnum
 from json import JSONDecodeError
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Protocol, Self
 
 from pydantic import TypeAdapter
 from sqlalchemy import select
@@ -41,6 +45,7 @@ from graphon.model_runtime.entities.provider_entities import (
     ProviderEntity,
 )
 from graphon.model_runtime.model_providers.model_provider_factory import ModelProviderFactory
+from models.enums import CredentialSourceType
 from models.provider import (
     LoadBalancingModelConfig,
     Provider,
@@ -59,7 +64,494 @@ if TYPE_CHECKING:
     from graphon.model_runtime.protocols.runtime import ModelRuntime
     from models.account import Account
 
+logger = logging.getLogger(__name__)
+
 _credentials_adapter: TypeAdapter[dict[str, Any]] = TypeAdapter(dict[str, Any])
+_PROVIDER_CONFIGURATION_CACHE_TTL_SECONDS = 300
+_PROVIDER_CONFIGURATION_CACHE_VERSION_TTL_SECONDS = 360
+_PROVIDER_CONFIGURATION_CACHE_VERSION_KEY = "provider_configurations:tenant:{tenant_id}:source:{source}:version"
+_PROVIDER_CONFIGURATION_CACHE_SOURCE_KEY = "provider_configurations:tenant:{tenant_id}:source:{source}:v:{version}"
+
+
+class ProviderConfigurationCacheSource(StrEnum):
+    PROVIDER_MODELS = "provider_models"
+    PREFERRED_MODEL_PROVIDERS = "preferred_model_providers"
+    PROVIDER_MODEL_SETTINGS = "provider_model_settings"
+    PROVIDER_MODEL_CREDENTIALS = "provider_model_credentials"
+    PROVIDER_CREDENTIALS = "provider_credentials"
+    PROVIDER_LOAD_BALANCING_CONFIGS = "provider_load_balancing_configs"
+
+
+_PROVIDER_CONFIGURATION_SOURCES = tuple(ProviderConfigurationCacheSource)
+
+
+class _CacheEntry(Protocol):
+    @classmethod
+    def from_cache_row(cls, row: dict[str, Any]) -> Self: ...
+
+    def to_cache_row(self) -> dict[str, Any]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderConfigurationCacheSourceSpec[T: _CacheEntry]:
+    name: ProviderConfigurationCacheSource
+    entry_cls: type[T]
+    load_records: Callable[[str], list[T]]
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderModelCacheEntry:
+    id: str
+    provider_name: str
+    model_name: str
+    model_type: ModelType
+    credential_id: str | None
+    credential_name: str | None
+    encrypted_config: str | None
+
+    @classmethod
+    def from_record(cls, record: ProviderModel) -> _ProviderModelCacheEntry:
+        credential = record.__dict__.get("credential")
+        return cls(
+            id=record.id,
+            provider_name=record.provider_name,
+            model_name=record.model_name,
+            model_type=record.model_type,
+            credential_id=record.credential_id,
+            credential_name=credential.credential_name if credential else None,
+            encrypted_config=credential.encrypted_config if credential else None,
+        )
+
+    @classmethod
+    def from_cache_row(cls, row: dict[str, Any]) -> _ProviderModelCacheEntry:
+        return cls(
+            id=row["id"],
+            provider_name=row["provider_name"],
+            model_name=row["model_name"],
+            model_type=ModelType(row["model_type"]),
+            credential_id=row.get("credential_id"),
+            credential_name=row.get("credential_name"),
+            encrypted_config=row.get("encrypted_config"),
+        )
+
+    def to_cache_row(self) -> dict[str, Any]:
+        row = asdict(self)
+        row["model_type"] = self.model_type.value
+        return row
+
+
+@dataclass(frozen=True, slots=True)
+class _TenantPreferredModelProviderCacheEntry:
+    provider_name: str
+    preferred_provider_type: ProviderType
+
+    @classmethod
+    def from_record(cls, record: TenantPreferredModelProvider) -> _TenantPreferredModelProviderCacheEntry:
+        return cls(
+            provider_name=record.provider_name,
+            preferred_provider_type=record.preferred_provider_type,
+        )
+
+    @classmethod
+    def from_cache_row(cls, row: dict[str, Any]) -> _TenantPreferredModelProviderCacheEntry:
+        return cls(
+            provider_name=row["provider_name"],
+            preferred_provider_type=ProviderType(row["preferred_provider_type"]),
+        )
+
+    def to_cache_row(self) -> dict[str, Any]:
+        return {
+            "provider_name": self.provider_name,
+            "preferred_provider_type": self.preferred_provider_type.value,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderModelSettingCacheEntry:
+    provider_name: str
+    model_name: str
+    model_type: ModelType
+    enabled: bool
+    load_balancing_enabled: bool
+
+    @classmethod
+    def from_record(cls, record: ProviderModelSetting) -> _ProviderModelSettingCacheEntry:
+        return cls(
+            provider_name=record.provider_name,
+            model_name=record.model_name,
+            model_type=record.model_type,
+            enabled=record.enabled,
+            load_balancing_enabled=record.load_balancing_enabled,
+        )
+
+    @classmethod
+    def from_cache_row(cls, row: dict[str, Any]) -> _ProviderModelSettingCacheEntry:
+        return cls(
+            provider_name=row["provider_name"],
+            model_name=row["model_name"],
+            model_type=ModelType(row["model_type"]),
+            enabled=row["enabled"],
+            load_balancing_enabled=row["load_balancing_enabled"],
+        )
+
+    def to_cache_row(self) -> dict[str, Any]:
+        return {
+            "provider_name": self.provider_name,
+            "model_name": self.model_name,
+            "model_type": self.model_type.value,
+            "enabled": self.enabled,
+            "load_balancing_enabled": self.load_balancing_enabled,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderModelCredentialCacheEntry:
+    id: str
+    provider_name: str
+    model_name: str
+    model_type: ModelType
+    credential_name: str
+
+    @classmethod
+    def from_record(cls, record: ProviderModelCredential) -> _ProviderModelCredentialCacheEntry:
+        return cls(
+            id=record.id,
+            provider_name=record.provider_name,
+            model_name=record.model_name,
+            model_type=record.model_type,
+            credential_name=record.credential_name,
+        )
+
+    @classmethod
+    def from_cache_row(cls, row: dict[str, Any]) -> _ProviderModelCredentialCacheEntry:
+        return cls(
+            id=row["id"],
+            provider_name=row["provider_name"],
+            model_name=row["model_name"],
+            model_type=ModelType(row["model_type"]),
+            credential_name=row["credential_name"],
+        )
+
+    def to_cache_row(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "provider_name": self.provider_name,
+            "model_name": self.model_name,
+            "model_type": self.model_type.value,
+            "credential_name": self.credential_name,
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class _ProviderCredentialCacheEntry:
+    id: str
+    provider_name: str
+    credential_name: str
+
+    @classmethod
+    def from_record(cls, record: ProviderCredential) -> _ProviderCredentialCacheEntry:
+        return cls(
+            id=record.id,
+            provider_name=record.provider_name,
+            credential_name=record.credential_name,
+        )
+
+    @classmethod
+    def from_cache_row(cls, row: dict[str, Any]) -> _ProviderCredentialCacheEntry:
+        return cls(
+            id=row["id"],
+            provider_name=row["provider_name"],
+            credential_name=row["credential_name"],
+        )
+
+    def to_cache_row(self) -> dict[str, Any]:
+        return asdict(self)
+
+
+@dataclass(frozen=True, slots=True)
+class _LoadBalancingModelConfigCacheEntry:
+    id: str
+    tenant_id: str
+    provider_name: str
+    model_name: str
+    model_type: ModelType
+    name: str
+    encrypted_config: str | None
+    credential_id: str | None
+    credential_source_type: CredentialSourceType | None
+    enabled: bool
+
+    @classmethod
+    def from_record(cls, record: LoadBalancingModelConfig) -> _LoadBalancingModelConfigCacheEntry:
+        return cls(
+            id=record.id,
+            tenant_id=record.tenant_id,
+            provider_name=record.provider_name,
+            model_name=record.model_name,
+            model_type=record.model_type,
+            name=record.name,
+            encrypted_config=record.encrypted_config,
+            credential_id=record.credential_id,
+            credential_source_type=record.credential_source_type,
+            enabled=record.enabled,
+        )
+
+    @classmethod
+    def from_cache_row(cls, row: dict[str, Any]) -> _LoadBalancingModelConfigCacheEntry:
+        return cls(
+            id=row["id"],
+            tenant_id=row["tenant_id"],
+            provider_name=row["provider_name"],
+            model_name=row["model_name"],
+            model_type=ModelType(row["model_type"]),
+            name=row["name"],
+            encrypted_config=row.get("encrypted_config"),
+            credential_id=row.get("credential_id"),
+            credential_source_type=CredentialSourceType(row["credential_source_type"])
+            if row.get("credential_source_type")
+            else None,
+            enabled=row["enabled"],
+        )
+
+    def to_cache_row(self) -> dict[str, Any]:
+        return {
+            "id": self.id,
+            "tenant_id": self.tenant_id,
+            "provider_name": self.provider_name,
+            "model_name": self.model_name,
+            "model_type": self.model_type.value,
+            "name": self.name,
+            "encrypted_config": self.encrypted_config,
+            "credential_id": self.credential_id,
+            "credential_source_type": self.credential_source_type.value if self.credential_source_type else None,
+            "enabled": self.enabled,
+        }
+
+
+class _ProviderConfigurationSourceCache:
+    """Redis-backed cache for tenant provider DB cache entries.
+
+    The assembled ``ProviderConfigurations`` object is intentionally not cached
+    here because it carries request-scoped runtime bindings. Cache only the DB
+    rows that are stable enough to reuse across processes, then let each
+    ``ProviderManager`` assemble and bind fresh runtime-aware entities.
+    """
+
+    @classmethod
+    def get_records[T: _CacheEntry](
+        cls,
+        *,
+        tenant_id: str,
+        source: ProviderConfigurationCacheSource,
+        entry_cls: type[T],
+    ) -> tuple[list[T] | None, str | None]:
+        version: str | None = None
+        try:
+            version = cls._get_version(tenant_id=tenant_id, source=source)
+            cache_key = cls._source_key(tenant_id=tenant_id, source=source, version=version)
+            cached_records = redis_client.get(cache_key)
+            if cached_records is None:
+                return None, version
+
+            cached_text = cached_records.decode("utf-8") if isinstance(cached_records, bytes) else cached_records
+            rows = json.loads(cached_text)
+            if not isinstance(rows, list):
+                return None, version
+
+            return [entry_cls.from_cache_row(row) for row in rows if isinstance(row, dict)], version
+        except Exception:
+            logger.warning("Failed to read provider configuration source cache", exc_info=True)
+            return None, version
+
+    @classmethod
+    def set_records(
+        cls,
+        *,
+        tenant_id: str,
+        source: ProviderConfigurationCacheSource,
+        records: Sequence[_CacheEntry],
+        expected_version: str | None = None,
+    ) -> None:
+        try:
+            version = cls._get_version(tenant_id=tenant_id, source=source)
+            if expected_version is not None and version != expected_version:
+                return
+            cache_key = cls._source_key(tenant_id=tenant_id, source=source, version=version)
+            rows = [record.to_cache_row() for record in records]
+            redis_client.setex(cache_key, _PROVIDER_CONFIGURATION_CACHE_TTL_SECONDS, json.dumps(rows))
+        except Exception:
+            logger.warning("Failed to write provider configuration source cache", exc_info=True)
+
+    @classmethod
+    def invalidate_tenant(
+        cls,
+        tenant_id: str,
+        sources: Sequence[ProviderConfigurationCacheSource] | None = None,
+    ) -> None:
+        try:
+            if sources is None:
+                sources = _PROVIDER_CONFIGURATION_SOURCES
+            for source in sources:
+                version_key = _PROVIDER_CONFIGURATION_CACHE_VERSION_KEY.format(tenant_id=tenant_id, source=source.value)
+                redis_client.incr(version_key)
+                redis_client.expire(version_key, _PROVIDER_CONFIGURATION_CACHE_VERSION_TTL_SECONDS)
+        except Exception:
+            logger.warning("Failed to invalidate provider configuration source cache", exc_info=True)
+
+    @classmethod
+    def _get_version(cls, *, tenant_id: str, source: ProviderConfigurationCacheSource) -> str:
+        version_key = _PROVIDER_CONFIGURATION_CACHE_VERSION_KEY.format(tenant_id=tenant_id, source=source.value)
+        version = redis_client.get(version_key)
+        if version is None:
+            redis_client.set(version_key, "0", ex=_PROVIDER_CONFIGURATION_CACHE_VERSION_TTL_SECONDS)
+            return "0"
+        redis_client.expire(version_key, _PROVIDER_CONFIGURATION_CACHE_VERSION_TTL_SECONDS)
+        return version.decode("utf-8") if isinstance(version, bytes) else str(version)
+
+    @staticmethod
+    def _source_key(*, tenant_id: str, source: ProviderConfigurationCacheSource, version: str) -> str:
+        return _PROVIDER_CONFIGURATION_CACHE_SOURCE_KEY.format(
+            tenant_id=tenant_id,
+            source=source.value,
+            version=version,
+        )
+
+
+def _get_cached_or_load_records[T: _CacheEntry](
+    *,
+    tenant_id: str,
+    cache_source: _ProviderConfigurationCacheSourceSpec[T],
+) -> list[T]:
+    cached_records, cache_version = _ProviderConfigurationSourceCache.get_records(
+        tenant_id=tenant_id,
+        source=cache_source.name,
+        entry_cls=cache_source.entry_cls,
+    )
+    if cached_records is not None:
+        return cached_records
+
+    records = cache_source.load_records(tenant_id)
+    _ProviderConfigurationSourceCache.set_records(
+        tenant_id=tenant_id,
+        source=cache_source.name,
+        records=records,
+        expected_version=cache_version,
+    )
+    return records
+
+
+def _attach_active_credentials(
+    *,
+    session: Any,
+    records: Sequence[Provider | ProviderModel],
+    credential_model_cls: type[ProviderCredential | ProviderModelCredential],
+) -> None:
+    credential_ids = [record.credential_id for record in records if getattr(record, "credential_id", None)]
+    if not credential_ids:
+        return
+
+    credentials = session.scalars(select(credential_model_cls).where(credential_model_cls.id.in_(credential_ids))).all()
+    credential_by_id = {credential.id: credential for credential in credentials}
+    for record in records:
+        if getattr(record, "credential_id", None):
+            record.__dict__["credential"] = credential_by_id.get(record.credential_id)
+
+
+def _load_provider_model_cache_entries(tenant_id: str) -> list[_ProviderModelCacheEntry]:
+    with session_factory.create_session() as session:
+        stmt = select(ProviderModel).where(ProviderModel.tenant_id == tenant_id, ProviderModel.is_valid == True)
+        provider_models = list(session.scalars(stmt))
+        _attach_active_credentials(
+            session=session,
+            records=provider_models,
+            credential_model_cls=ProviderModelCredential,
+        )
+        return [_ProviderModelCacheEntry.from_record(provider_model) for provider_model in provider_models]
+
+
+def _load_preferred_model_provider_cache_entries(tenant_id: str) -> list[_TenantPreferredModelProviderCacheEntry]:
+    with session_factory.create_session() as session:
+        stmt = select(TenantPreferredModelProvider).where(TenantPreferredModelProvider.tenant_id == tenant_id)
+        return [
+            _TenantPreferredModelProviderCacheEntry.from_record(preferred_model_provider)
+            for preferred_model_provider in session.scalars(stmt)
+        ]
+
+
+def _load_provider_model_setting_cache_entries(tenant_id: str) -> list[_ProviderModelSettingCacheEntry]:
+    with session_factory.create_session() as session:
+        stmt = select(ProviderModelSetting).where(ProviderModelSetting.tenant_id == tenant_id)
+        return [
+            _ProviderModelSettingCacheEntry.from_record(provider_model_setting)
+            for provider_model_setting in session.scalars(stmt)
+        ]
+
+
+def _load_provider_model_credential_cache_entries(tenant_id: str) -> list[_ProviderModelCredentialCacheEntry]:
+    with session_factory.create_session() as session:
+        stmt = (
+            select(ProviderModelCredential)
+            .where(ProviderModelCredential.tenant_id == tenant_id)
+            .order_by(ProviderModelCredential.created_at.desc())
+        )
+        return [
+            _ProviderModelCredentialCacheEntry.from_record(provider_model_credential)
+            for provider_model_credential in session.scalars(stmt)
+        ]
+
+
+def _load_provider_credential_cache_entries(tenant_id: str) -> list[_ProviderCredentialCacheEntry]:
+    with session_factory.create_session() as session:
+        stmt = (
+            select(ProviderCredential)
+            .where(ProviderCredential.tenant_id == tenant_id)
+            .order_by(ProviderCredential.created_at.desc())
+        )
+        return [
+            _ProviderCredentialCacheEntry.from_record(provider_credential)
+            for provider_credential in session.scalars(stmt)
+        ]
+
+
+def _load_provider_load_balancing_config_cache_entries(tenant_id: str) -> list[_LoadBalancingModelConfigCacheEntry]:
+    with session_factory.create_session() as session:
+        stmt = select(LoadBalancingModelConfig).where(LoadBalancingModelConfig.tenant_id == tenant_id)
+        return [
+            _LoadBalancingModelConfigCacheEntry.from_record(load_balancing_model_config)
+            for load_balancing_model_config in session.scalars(stmt)
+        ]
+
+
+_PROVIDER_MODELS_CACHE_SOURCE = _ProviderConfigurationCacheSourceSpec(
+    name=ProviderConfigurationCacheSource.PROVIDER_MODELS,
+    entry_cls=_ProviderModelCacheEntry,
+    load_records=_load_provider_model_cache_entries,
+)
+_PREFERRED_MODEL_PROVIDERS_CACHE_SOURCE = _ProviderConfigurationCacheSourceSpec(
+    name=ProviderConfigurationCacheSource.PREFERRED_MODEL_PROVIDERS,
+    entry_cls=_TenantPreferredModelProviderCacheEntry,
+    load_records=_load_preferred_model_provider_cache_entries,
+)
+_PROVIDER_MODEL_SETTINGS_CACHE_SOURCE = _ProviderConfigurationCacheSourceSpec(
+    name=ProviderConfigurationCacheSource.PROVIDER_MODEL_SETTINGS,
+    entry_cls=_ProviderModelSettingCacheEntry,
+    load_records=_load_provider_model_setting_cache_entries,
+)
+_PROVIDER_MODEL_CREDENTIALS_CACHE_SOURCE = _ProviderConfigurationCacheSourceSpec(
+    name=ProviderConfigurationCacheSource.PROVIDER_MODEL_CREDENTIALS,
+    entry_cls=_ProviderModelCredentialCacheEntry,
+    load_records=_load_provider_model_credential_cache_entries,
+)
+_PROVIDER_CREDENTIALS_CACHE_SOURCE = _ProviderConfigurationCacheSourceSpec(
+    name=ProviderConfigurationCacheSource.PROVIDER_CREDENTIALS,
+    entry_cls=_ProviderCredentialCacheEntry,
+    load_records=_load_provider_credential_cache_entries,
+)
+_PROVIDER_LOAD_BALANCING_CONFIGS_CACHE_SOURCE = _ProviderConfigurationCacheSourceSpec(
+    name=ProviderConfigurationCacheSource.PROVIDER_LOAD_BALANCING_CONFIGS,
+    entry_cls=_LoadBalancingModelConfigCacheEntry,
+    load_records=_load_provider_load_balancing_config_cache_entries,
+)
 
 
 class ProviderManager:
@@ -98,6 +590,14 @@ class ProviderManager:
 
         self._configurations_cache.pop(tenant_id, None)
 
+    @staticmethod
+    def invalidate_configurations_cache(
+        tenant_id: str,
+        sources: Sequence[ProviderConfigurationCacheSource] | None = None,
+    ) -> None:
+        """Invalidate cross-process provider configuration source cache for a tenant."""
+        _ProviderConfigurationSourceCache.invalidate_tenant(tenant_id, sources=sources)
+
     def get_configurations(self, tenant_id: str) -> ProviderConfigurations:
         """
         Get model provider configurations.
@@ -192,6 +692,9 @@ class ProviderManager:
         # Get All provider model credentials
         provider_name_to_provider_model_credentials_dict = self._get_all_provider_model_credentials(tenant_id)
 
+        # Get All provider credentials
+        provider_name_to_provider_credentials_dict = self._get_all_provider_credentials(tenant_id)
+
         provider_configurations = ProviderConfigurations(tenant_id=tenant_id)
 
         # Construct ProviderConfiguration objects for each provider
@@ -224,7 +727,12 @@ class ProviderManager:
 
             # Convert to custom configuration
             custom_configuration = self._to_custom_configuration(
-                tenant_id, provider_entity, provider_records, provider_model_records, provider_model_credentials
+                tenant_id,
+                provider_entity,
+                provider_records,
+                provider_model_records,
+                provider_model_credentials,
+                provider_name_to_provider_credentials_dict,
             )
 
             # Convert to system configuration
@@ -448,84 +956,115 @@ class ProviderManager:
         provider_name_to_provider_records_dict = defaultdict(list)
         with session_factory.create_session() as session:
             stmt = select(Provider).where(Provider.tenant_id == tenant_id, Provider.is_valid == True)
-            providers = session.scalars(stmt)
+            providers = list(session.scalars(stmt))
+            _attach_active_credentials(
+                session=session,
+                records=providers,
+                credential_model_cls=ProviderCredential,
+            )
             for provider in providers:
                 # Use provider name with prefix after the data migration
                 provider_name_to_provider_records_dict[str(ModelProviderID(provider.provider_name))].append(provider)
         return provider_name_to_provider_records_dict
 
     @staticmethod
-    def _get_all_provider_models(tenant_id: str) -> dict[str, list[ProviderModel]]:
+    def _get_all_provider_models(tenant_id: str) -> dict[str, list[_ProviderModelCacheEntry]]:
         """
         Get all provider model records of the workspace.
 
         :param tenant_id: workspace id
         :return:
         """
+        provider_models = _get_cached_or_load_records(
+            tenant_id=tenant_id,
+            cache_source=_PROVIDER_MODELS_CACHE_SOURCE,
+        )
+
         provider_name_to_provider_model_records_dict = defaultdict(list)
-        with session_factory.create_session() as session:
-            stmt = select(ProviderModel).where(ProviderModel.tenant_id == tenant_id, ProviderModel.is_valid == True)
-            provider_models = session.scalars(stmt)
-            for provider_model in provider_models:
-                provider_name_to_provider_model_records_dict[provider_model.provider_name].append(provider_model)
+        for provider_model in provider_models:
+            provider_name_to_provider_model_records_dict[provider_model.provider_name].append(provider_model)
         return provider_name_to_provider_model_records_dict
 
     @staticmethod
-    def _get_all_preferred_model_providers(tenant_id: str) -> dict[str, TenantPreferredModelProvider]:
+    def _get_all_preferred_model_providers(tenant_id: str) -> dict[str, _TenantPreferredModelProviderCacheEntry]:
         """
         Get All preferred provider types of the workspace.
 
         :param tenant_id: workspace id
         :return:
         """
-        provider_name_to_preferred_provider_type_records_dict = {}
-        with session_factory.create_session() as session:
-            stmt = select(TenantPreferredModelProvider).where(TenantPreferredModelProvider.tenant_id == tenant_id)
-            preferred_provider_types = session.scalars(stmt)
-            provider_name_to_preferred_provider_type_records_dict = {
-                preferred_provider_type.provider_name: preferred_provider_type
-                for preferred_provider_type in preferred_provider_types
-            }
-        return provider_name_to_preferred_provider_type_records_dict
+        preferred_provider_types = _get_cached_or_load_records(
+            tenant_id=tenant_id,
+            cache_source=_PREFERRED_MODEL_PROVIDERS_CACHE_SOURCE,
+        )
+
+        return {
+            preferred_provider_type.provider_name: preferred_provider_type
+            for preferred_provider_type in preferred_provider_types
+        }
 
     @staticmethod
-    def _get_all_provider_model_settings(tenant_id: str) -> dict[str, list[ProviderModelSetting]]:
+    def _get_all_provider_model_settings(tenant_id: str) -> dict[str, list[_ProviderModelSettingCacheEntry]]:
         """
         Get All provider model settings of the workspace.
 
         :param tenant_id: workspace id
         :return:
         """
+        provider_model_settings = _get_cached_or_load_records(
+            tenant_id=tenant_id,
+            cache_source=_PROVIDER_MODEL_SETTINGS_CACHE_SOURCE,
+        )
+
         provider_name_to_provider_model_settings_dict = defaultdict(list)
-        with session_factory.create_session() as session:
-            stmt = select(ProviderModelSetting).where(ProviderModelSetting.tenant_id == tenant_id)
-            provider_model_settings = session.scalars(stmt)
-            for provider_model_setting in provider_model_settings:
-                provider_name_to_provider_model_settings_dict[provider_model_setting.provider_name].append(
-                    provider_model_setting
-                )
+        for provider_model_setting in provider_model_settings:
+            provider_name_to_provider_model_settings_dict[provider_model_setting.provider_name].append(
+                provider_model_setting
+            )
         return provider_name_to_provider_model_settings_dict
 
     @staticmethod
-    def _get_all_provider_model_credentials(tenant_id: str) -> dict[str, list[ProviderModelCredential]]:
+    def _get_all_provider_model_credentials(tenant_id: str) -> dict[str, list[_ProviderModelCredentialCacheEntry]]:
         """
         Get All provider model credentials of the workspace.
 
         :param tenant_id: workspace id
         :return:
         """
+        provider_model_credentials = _get_cached_or_load_records(
+            tenant_id=tenant_id,
+            cache_source=_PROVIDER_MODEL_CREDENTIALS_CACHE_SOURCE,
+        )
+
         provider_name_to_provider_model_credentials_dict = defaultdict(list)
-        with session_factory.create_session() as session:
-            stmt = select(ProviderModelCredential).where(ProviderModelCredential.tenant_id == tenant_id)
-            provider_model_credentials = session.scalars(stmt)
-            for provider_model_credential in provider_model_credentials:
-                provider_name_to_provider_model_credentials_dict[provider_model_credential.provider_name].append(
-                    provider_model_credential
-                )
+        for provider_model_credential in provider_model_credentials:
+            provider_name_to_provider_model_credentials_dict[provider_model_credential.provider_name].append(
+                provider_model_credential
+            )
         return provider_name_to_provider_model_credentials_dict
 
     @staticmethod
-    def _get_all_provider_load_balancing_configs(tenant_id: str) -> dict[str, list[LoadBalancingModelConfig]]:
+    def _get_all_provider_credentials(tenant_id: str) -> dict[str, list[_ProviderCredentialCacheEntry]]:
+        """
+        Get All provider credentials of the workspace.
+
+        :param tenant_id: workspace id
+        :return:
+        """
+        provider_credentials = _get_cached_or_load_records(
+            tenant_id=tenant_id,
+            cache_source=_PROVIDER_CREDENTIALS_CACHE_SOURCE,
+        )
+
+        provider_name_to_provider_credentials_dict = defaultdict(list)
+        for provider_credential in provider_credentials:
+            provider_name_to_provider_credentials_dict[provider_credential.provider_name].append(provider_credential)
+        return provider_name_to_provider_credentials_dict
+
+    @staticmethod
+    def _get_all_provider_load_balancing_configs(
+        tenant_id: str,
+    ) -> dict[str, list[_LoadBalancingModelConfigCacheEntry]]:
         """
         Get All provider load balancing configs of the workspace.
 
@@ -546,14 +1085,16 @@ class ProviderManager:
         if not model_load_balancing_enabled:
             return {}
 
+        provider_load_balancing_configs = _get_cached_or_load_records(
+            tenant_id=tenant_id,
+            cache_source=_PROVIDER_LOAD_BALANCING_CONFIGS_CACHE_SOURCE,
+        )
+
         provider_name_to_provider_load_balancing_model_configs_dict = defaultdict(list)
-        with session_factory.create_session() as session:
-            stmt = select(LoadBalancingModelConfig).where(LoadBalancingModelConfig.tenant_id == tenant_id)
-            provider_load_balancing_configs = session.scalars(stmt)
-            for provider_load_balancing_config in provider_load_balancing_configs:
-                provider_name_to_provider_load_balancing_model_configs_dict[
-                    provider_load_balancing_config.provider_name
-                ].append(provider_load_balancing_config)
+        for provider_load_balancing_config in provider_load_balancing_configs:
+            provider_name_to_provider_load_balancing_model_configs_dict[
+                provider_load_balancing_config.provider_name
+            ].append(provider_load_balancing_config)
 
         return provider_name_to_provider_load_balancing_model_configs_dict
 
@@ -722,8 +1263,9 @@ class ProviderManager:
         tenant_id: str,
         provider_entity: ProviderEntity,
         provider_records: list[Provider],
-        provider_model_records: list[ProviderModel],
-        provider_model_credentials: list[ProviderModelCredential],
+        provider_model_records: list[_ProviderModelCacheEntry],
+        provider_model_credentials: list[_ProviderModelCredentialCacheEntry],
+        provider_credentials_by_name: dict[str, list[_ProviderCredentialCacheEntry]],
     ) -> CustomConfiguration:
         """
         Convert to custom configuration.
@@ -736,7 +1278,10 @@ class ProviderManager:
         """
         # Get custom provider configuration
         custom_provider_configuration = self._get_custom_provider_configuration(
-            tenant_id, provider_entity, provider_records
+            tenant_id,
+            provider_entity,
+            provider_records,
+            provider_credentials_by_name,
         )
 
         # Get custom models which have not been added to the model list yet
@@ -758,7 +1303,11 @@ class ProviderManager:
         )
 
     def _get_custom_provider_configuration(
-        self, tenant_id: str, provider_entity: ProviderEntity, provider_records: list[Provider]
+        self,
+        tenant_id: str,
+        provider_entity: ProviderEntity,
+        provider_records: list[Provider],
+        provider_credentials_by_name: dict[str, list[_ProviderCredentialCacheEntry]],
     ) -> CustomProviderConfiguration | None:
         """Get custom provider configuration."""
         # Find custom provider record (non-system)
@@ -790,13 +1339,29 @@ class ProviderManager:
             credentials=provider_credentials,
             current_credential_name=custom_provider_record.credential_name,
             current_credential_id=custom_provider_record.credential_id,
-            available_credentials=self.get_provider_available_credentials(
-                tenant_id, custom_provider_record.provider_name
+            available_credentials=self._get_provider_available_credentials_from_records(
+                custom_provider_record.provider_name,
+                provider_credentials_by_name,
             ),
         )
 
+    @staticmethod
+    def _get_provider_available_credentials_from_records(
+        provider_name: str,
+        provider_credentials_by_name: dict[str, list[_ProviderCredentialCacheEntry]],
+    ) -> list[CredentialConfiguration]:
+        available_credentials: list[CredentialConfiguration] = []
+        for candidate_provider_name in ProviderManager._get_provider_names(provider_name):
+            available_credentials.extend(
+                CredentialConfiguration(credential_id=credential.id, credential_name=credential.credential_name)
+                for credential in provider_credentials_by_name.get(candidate_provider_name, [])
+            )
+        return available_credentials
+
     def _get_can_added_models(
-        self, provider_model_records: list[ProviderModel], all_model_credentials: Sequence[ProviderModelCredential]
+        self,
+        provider_model_records: list[_ProviderModelCacheEntry],
+        all_model_credentials: Sequence[_ProviderModelCredentialCacheEntry],
     ) -> list[dict]:
         """Get the custom models and credentials from enterprise version which haven't add to the model list"""
         existing_model_set = {(record.model_name, record.model_type) for record in provider_model_records}
@@ -829,9 +1394,9 @@ class ProviderManager:
         self,
         tenant_id: str,
         provider_entity: ProviderEntity,
-        provider_model_records: list[ProviderModel],
+        provider_model_records: list[_ProviderModelCacheEntry],
         can_added_models: list[dict],
-        all_model_credentials: Sequence[ProviderModelCredential],
+        all_model_credentials: Sequence[_ProviderModelCredentialCacheEntry],
     ) -> list[CustomModelConfiguration]:
         """Get custom model configurations."""
         # Get model credential secret variables
@@ -1151,8 +1716,8 @@ class ProviderManager:
     def _to_model_settings(
         self,
         provider_entity: ProviderEntity,
-        provider_model_settings: list[ProviderModelSetting] | None = None,
-        load_balancing_model_configs: list[LoadBalancingModelConfig] | None = None,
+        provider_model_settings: list[_ProviderModelSettingCacheEntry] | None = None,
+        load_balancing_model_configs: list[_LoadBalancingModelConfigCacheEntry] | None = None,
     ) -> list[ModelSettings]:
         """
         Convert to model settings.

api/core/rag/retrieval/dataset_retrieval.py

@@ -1030,6 +1030,10 @@ class DatasetRetrieval:
     ):
         """
         Persist dataset query audit rows for retrieval requests.
+
+        Query audit logging is a side effect of retrieval. Keep it in an
+        independent transaction so failures or commits here do not affect the
+        request/workflow transaction that called the retriever.
         """
         if not query and not attachment_ids:
             return
@@ -1041,6 +1045,9 @@ class DatasetRetrieval:
                 app_id,
             )
             return
+        created_by_role = self._resolve_creator_user_role(user_from)
+        if created_by_role is None:
+            return
         dataset_queries = []
         for dataset_id in dataset_ids:
             contents = []
@@ -1055,13 +1062,16 @@ class DatasetRetrieval:
                     content=json.dumps(contents),
                     source=DatasetQuerySource.APP,
                     source_app_id=app_id,
-                    created_by_role=CreatorUserRole(user_from),
+                    created_by_role=created_by_role,
                     created_by=created_by,
                 )
                 dataset_queries.append(dataset_query)
-            if dataset_queries:
-                db.session.add_all(dataset_queries)
-        db.session.commit()
+
+        if not dataset_queries:
+            return
+
+        with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session:
+            session.add_all(dataset_queries)
 
     def _retriever(
         self,

api/core/rbac/__init__.py

@@ -1,3 +1,3 @@
-from core.rbac.entities import RBACPermission, RBACResourceScope
+from core.rbac.entities import RBACPermission, RBACResourceScope, RBACResourceWhitelistScope
 
-__all__ = ["RBACPermission", "RBACResourceScope"]
+__all__ = ["RBACPermission", "RBACResourceScope", "RBACResourceWhitelistScope"]

api/core/rbac/entities.py

@@ -13,6 +13,14 @@ class RBACResourceScope(StrEnum):
     WORKSPACE = "workspace"
 
 
+class RBACResourceWhitelistScope(StrEnum):
+    """Whitelist scopes accepted by RBAC app and dataset access config APIs."""
+
+    ALL = "all"
+    SPECIFIC = "specific"
+    ONLY_ME = "only_me"
+
+
 class RBACPermission(StrEnum):
     """Permission points (RBAC scenes) checked by ``rbac_permission_required``.
 

api/core/tools/tool_engine.py

@@ -7,6 +7,7 @@ from datetime import UTC, datetime
 from mimetypes import guess_type
 from typing import Any, Union, cast
 
+from sqlalchemy.orm import sessionmaker
 from yarl import URL
 
 from core.app.entities.app_invoke_entities import InvokeFrom
@@ -110,7 +111,7 @@ class ToolEngine:
                 tool_messages=binary_files, agent_message=message, invoke_from=invoke_from, user_id=user_id
             )
 
-            plain_text = ToolEngine._convert_tool_response_to_str(message_list)
+            plain_text = ToolEngine.tool_response_to_str(message_list)
 
             meta = invocation_meta_dict["meta"]
 
@@ -233,10 +234,8 @@ class ToolEngine:
             yield meta
 
     @staticmethod
-    def _convert_tool_response_to_str(tool_response: list[ToolInvokeMessage]) -> str:
-        """
-        Handle tool response
-        """
+    def tool_response_to_str(tool_response: list[ToolInvokeMessage]) -> str:
+        """Convert tool invoke messages into the plain-text observation shown to the model/user."""
         parts: list[str] = []
         json_parts: list[str] = []
 
@@ -338,47 +337,49 @@ class ToolEngine:
         user_id: str,
     ) -> list[str]:
         """
-        Create message file
+        Create message files produced by a tool call.
+
+        Tool file persistence is a side effect of agent execution. Use an
+        independent transaction so this helper never commits or closes the
+        caller's request-scoped session.
 
         :return: message file ids
         """
         result = []
 
-        for message in tool_messages:
-            if "image" in message.mimetype:
-                file_type = FileType.IMAGE
-            elif "video" in message.mimetype:
-                file_type = FileType.VIDEO
-            elif "audio" in message.mimetype:
-                file_type = FileType.AUDIO
-            elif "text" in message.mimetype or "pdf" in message.mimetype:
-                file_type = FileType.DOCUMENT
-            else:
-                file_type = FileType.CUSTOM
+        with sessionmaker(bind=db.engine, expire_on_commit=False).begin() as session:
+            for message in tool_messages:
+                # extract tool file id from url
+                tool_file_id = message.url.split("/")[-1].split(".")[0]
+                message_file = MessageFile(
+                    message_id=agent_message.id,
+                    type=ToolEngine._resolve_tool_file_type(message),
+                    transfer_method=FileTransferMethod.TOOL_FILE,
+                    belongs_to=MessageFileBelongsTo.ASSISTANT,
+                    url=message.url,
+                    upload_file_id=tool_file_id,
+                    created_by_role=(
+                        CreatorUserRole.ACCOUNT
+                        if invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER}
+                        else CreatorUserRole.END_USER
+                    ),
+                    created_by=user_id,
+                )
 
-            # extract tool file id from url
-            tool_file_id = message.url.split("/")[-1].split(".")[0]
-            message_file = MessageFile(
-                message_id=agent_message.id,
-                type=file_type,
-                transfer_method=FileTransferMethod.TOOL_FILE,
-                belongs_to=MessageFileBelongsTo.ASSISTANT,
-                url=message.url,
-                upload_file_id=tool_file_id,
-                created_by_role=(
-                    CreatorUserRole.ACCOUNT
-                    if invoke_from in {InvokeFrom.EXPLORE, InvokeFrom.DEBUGGER}
-                    else CreatorUserRole.END_USER
-                ),
-                created_by=user_id,
-            )
-
-            db.session.add(message_file)
-            db.session.commit()
-            db.session.refresh(message_file)
-
-            result.append(message_file.id)
-
-        db.session.close()
+                session.add(message_file)
+                result.append(message_file.id)
 
         return result
+
+    @staticmethod
+    def _resolve_tool_file_type(message: ToolInvokeMessageBinary) -> FileType:
+        if "image" in message.mimetype:
+            return FileType.IMAGE
+        elif "video" in message.mimetype:
+            return FileType.VIDEO
+        elif "audio" in message.mimetype:
+            return FileType.AUDIO
+        elif "text" in message.mimetype or "pdf" in message.mimetype:
+            return FileType.DOCUMENT
+        else:
+            return FileType.CUSTOM

api/core/tools/utils/dataset_retriever/dataset_multi_retriever_tool.py

@@ -79,7 +79,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):
         all_documents = rerank_runner.run(query, all_documents, self.score_threshold, self.top_k)
 
         for hit_callback in self.hit_callbacks:
-            hit_callback.on_tool_end(all_documents)
+            hit_callback.on_tool_end(all_documents, db.session)
 
         document_score_list = {}
         for item in all_documents:
@@ -166,7 +166,7 @@ class DatasetMultiRetrieverTool(DatasetRetrieverBaseTool):
                 return []
 
             for hit_callback in hit_callbacks:
-                hit_callback.on_query(query, dataset.id)
+                hit_callback.on_query(query, dataset.id, db.session)
 
             # get retrieval model , if the model is not setting , using default
             retrieval_model = dataset.retrieval_model or default_retrieval_model

api/core/tools/utils/dataset_retriever/dataset_retriever_tool.py

@@ -64,7 +64,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool):
         if not dataset:
             return ""
         for hit_callback in self.hit_callbacks:
-            hit_callback.on_query(query, dataset.id)
+            hit_callback.on_query(query, dataset.id, db.session)
         dataset_retrieval = DatasetRetrieval()
         metadata_filter_document_ids, metadata_condition = dataset_retrieval.get_metadata_filter_condition(
             [dataset.id],
@@ -159,7 +159,7 @@ class DatasetRetrieverTool(DatasetRetrieverBaseTool):
                 else:
                     documents = []
                 for hit_callback in self.hit_callbacks:
-                    hit_callback.on_tool_end(documents)
+                    hit_callback.on_tool_end(documents, db.session)
                 document_score_list = {}
                 if dataset.indexing_technique != IndexTechniqueType.ECONOMY:
                     for item in documents:

api/core/workflow/generator/runner.py

@@ -27,6 +27,7 @@ import json
 import logging
 import re
 import time
+from collections.abc import Iterator
 from typing import Any, ClassVar, cast
 
 import json_repair
@@ -185,6 +186,48 @@ def _result_with_errors(
     return base
 
 
+def _with_mode(result: WorkflowGenerateResultDict, mode: WorkflowGenerationMode) -> WorkflowGenerateResultDict:
+    """Stamp the resolved concrete ``mode`` onto a result envelope.
+
+    ``mode="auto"`` requests are resolved to a concrete mode before planning;
+    echoing it back lets the frontend pick the right app type to create. It's
+    present for explicit modes too so the response shape stays uniform.
+    """
+    result["mode"] = mode
+    return result
+
+
+def _build_plan_event(
+    *,
+    plan: PlannerResultDict,
+    plan_nodes: list[dict[str, Any]],
+    start_inputs: list[dict[str, Any]],
+    mode: WorkflowGenerationMode,
+) -> dict[str, Any]:
+    """Shape the ``plan`` event emitted before the (slower) builder runs.
+
+    Node fields are pulled defensively: the planner schema only guarantees
+    ``node_type`` is present, so ``label`` / ``purpose`` may be missing on a
+    terse plan and default to empty strings.
+    """
+    return {
+        "title": str(plan.get("title") or ""),
+        "description": str(plan.get("description") or ""),
+        "app_name": str(plan.get("app_name") or "").strip(),
+        "icon": str(plan.get("icon") or "").strip(),
+        "mode": mode,
+        "nodes": [
+            {
+                "label": str(node.get("label") or ""),
+                "node_type": str(node.get("node_type") or ""),
+                "purpose": str(node.get("purpose") or ""),
+            }
+            for node in plan_nodes
+        ],
+        "start_inputs": start_inputs,
+    }
+
+
 def _stage_error_to_envelope_code(exc: Exception) -> str:
     """Map a stage-typed exception to the result envelope's error code."""
     if isinstance(exc, _StageJSONError):
@@ -250,6 +293,100 @@ class WorkflowGenerator:
         ``errors`` and keep the previous version visible.
         """
 
+        # Consume the shared event generator and keep only the final result
+        # envelope — ``generate_workflow_graph_stream`` shares the exact same
+        # pipeline so the two stay behaviourally identical. The plan event is
+        # ignored here.
+        result: WorkflowGenerateResultDict | None = None
+        for event_name, payload in cls._iter_generation_events(
+            model_instance=model_instance,
+            model_parameters=model_parameters,
+            provider=provider,
+            model_name=model_name,
+            model_mode=model_mode,
+            mode=mode,
+            instruction=instruction,
+            ideal_output=ideal_output,
+            tool_catalogue_text=tool_catalogue_text,
+            installed_tools=installed_tools,
+            current_graph=current_graph,
+        ):
+            if event_name == "result":
+                result = cast(WorkflowGenerateResultDict, payload)
+        # The event generator always emits exactly one result envelope; this
+        # fallback only guards against a future refactor that forgets to.
+        if result is None:
+            result = _with_mode(_empty_result(), mode)
+        return result
+
+    @classmethod
+    def generate_workflow_graph_stream(
+        cls,
+        *,
+        model_instance,
+        model_parameters: dict[str, Any],
+        provider: str,
+        model_name: str,
+        model_mode: str,
+        mode: WorkflowGenerationMode,
+        instruction: str,
+        ideal_output: str = "",
+        tool_catalogue_text: str = "",
+        installed_tools: set[tuple[str, str]] | None = None,
+        current_graph: dict[str, Any] | None = None,
+    ) -> Iterator[tuple[str, dict[str, Any]]]:
+        """
+        Streaming sibling of ``generate_workflow_graph``.
+
+        Yields a ``plan`` event (title / description / app_name / icon / mode /
+        high-level nodes / start_inputs) as soon as the planner returns, then a
+        final ``result`` event carrying the SAME envelope dict the non-streaming
+        method returns (graph / message / app_name / icon / error / errors /
+        mode, plus structural errors when any). On a planner / empty-plan /
+        builder failure only the ``result`` event is emitted — no ``plan``.
+        """
+        yield from cls._iter_generation_events(
+            model_instance=model_instance,
+            model_parameters=model_parameters,
+            provider=provider,
+            model_name=model_name,
+            model_mode=model_mode,
+            mode=mode,
+            instruction=instruction,
+            ideal_output=ideal_output,
+            tool_catalogue_text=tool_catalogue_text,
+            installed_tools=installed_tools,
+            current_graph=current_graph,
+        )
+
+    @classmethod
+    def _iter_generation_events(
+        cls,
+        *,
+        model_instance,
+        model_parameters: dict[str, Any],
+        provider: str,
+        model_name: str,
+        model_mode: str,
+        mode: WorkflowGenerationMode,
+        instruction: str,
+        ideal_output: str = "",
+        tool_catalogue_text: str = "",
+        installed_tools: set[tuple[str, str]] | None = None,
+        current_graph: dict[str, Any] | None = None,
+    ) -> Iterator[tuple[str, dict[str, Any]]]:
+        """
+        Drive planner → builder → postprocess and yield generation events.
+
+        Shared core for both ``generate_workflow_graph`` (keeps only the final
+        ``result``) and ``generate_workflow_graph_stream`` (streams every
+        event). Emits at most one ``plan`` event — only once the planner
+        produced a non-empty plan — followed by exactly one ``result`` event.
+        On a planner / empty-plan / builder failure it emits only the
+        ``result`` event carrying the error envelope. Every result envelope is
+        stamped with the resolved concrete ``mode``.
+        """
+
         # ── 1. PLANNER ────────────────────────────────────────────────────
         plan, plan_err = cls._run_stage(
             stage="Planner",
@@ -265,16 +402,22 @@ class WorkflowGenerator:
             ),
         )
         if plan_err is not None:
-            return _result_with_errors(_empty_result(), [plan_err])
+            yield "result", cast(dict[str, Any], _with_mode(_result_with_errors(_empty_result(), [plan_err]), mode))
+            return
 
         # The lambda return is non-None when no error fired — narrow it for type-checkers.
         plan = cast(PlannerResultDict, plan)
         plan_nodes: list[dict[str, Any]] = cast(list[dict[str, Any]], plan.get("nodes", []))
         if not plan_nodes:
-            return _result_with_errors(
-                _empty_result(),
-                [_err(WorkflowGenerateErrorCode.EMPTY_PLAN, "Planner returned no nodes")],
+            empty_plan = _with_mode(
+                _result_with_errors(
+                    _empty_result(),
+                    [_err(WorkflowGenerateErrorCode.EMPTY_PLAN, "Planner returned no nodes")],
+                ),
+                mode,
             )
+            yield "result", cast(dict[str, Any], empty_plan)
+            return
 
         # Planner-supplied user-input declarations. The builder uses these to
         # populate ``start.data.variables`` so downstream ``{#start.<var>#}``
@@ -286,6 +429,10 @@ class WorkflowGenerator:
             if isinstance(item, dict) and (item.get("variable") or "").strip()
         ]
 
+        # First event the stream sees: the high-level plan, before the slower
+        # builder call. Non-streaming callers ignore it.
+        yield "plan", _build_plan_event(plan=plan, plan_nodes=plan_nodes, start_inputs=start_inputs, mode=mode)
+
         # ── 2. BUILDER ────────────────────────────────────────────────────
         graph, build_err = cls._run_stage(
             stage="Builder",
@@ -306,7 +453,8 @@ class WorkflowGenerator:
             ),
         )
         if build_err is not None:
-            return _result_with_errors(_empty_result(), [build_err])
+            yield "result", cast(dict[str, Any], _with_mode(_result_with_errors(_empty_result(), [build_err]), mode))
+            return
         graph = cast(GraphDict, graph)
 
         # ── 3. POSTPROC + VALIDATE ────────────────────────────────────────
@@ -322,6 +470,7 @@ class WorkflowGenerator:
             "error": "",
             "errors": [],
         }
+        _with_mode(result, mode)
 
         # Final structural sanity check — fail closed if start/end shape is
         # wrong, container topology is broken, a tool was hallucinated, or a
@@ -330,8 +479,9 @@ class WorkflowGenerator:
         structural_errors = cls._validate_structure(graph=graph, mode=mode, installed_tools=installed_tools)
         if structural_errors:
             logger.warning("Workflow generator: structural validation failed: %s", structural_errors)
-            return _result_with_errors(result, structural_errors)
-        return result
+            yield "result", cast(dict[str, Any], _result_with_errors(result, structural_errors))
+            return
+        yield "result", cast(dict[str, Any], result)
 
     @classmethod
     def _run_stage(

api/core/workflow/generator/types.py

@@ -11,6 +11,13 @@ from typing import Final, Literal, NotRequired, TypedDict
 
 WorkflowGenerationMode = Literal["workflow", "advanced-chat"]
 
+# The mode accepted at the API boundary. ``auto`` is a sentinel that asks the
+# service to classify the instruction into a concrete ``WorkflowGenerationMode``
+# (one tiny LLM call) BEFORE planning — see
+# ``WorkflowGeneratorService._resolve_mode`` and
+# ``LLMGenerator.classify_workflow_mode``.
+WorkflowGenerationModeRequest = Literal["workflow", "advanced-chat", "auto"]
+
 
 # Machine-readable error codes returned in ``WorkflowGenerateResultDict.errors``.
 # Frontend maps these to localised copy via ``workflow.generator.errors.<code>``
@@ -148,3 +155,7 @@ class WorkflowGenerateResultDict(TypedDict):
     icon: str
     error: str
     errors: list[WorkflowGenerateErrorDict]
+    # Resolved concrete generation mode ("workflow" / "advanced-chat"). Stamped
+    # onto every envelope so a ``mode="auto"`` request can tell the frontend
+    # which app type to create; present for explicit modes too for uniformity.
+    mode: NotRequired[str]

api/core/workflow/nodes/agent_v2/agent_node.py

@@ -31,6 +31,7 @@ from graphon.enums import BuiltinNodeTypes, WorkflowNodeExecutionMetadataKey, Wo
 from graphon.node_events import NodeEventBase, NodeRunResult, PauseRequestedEvent, StreamCompletedEvent
 from graphon.nodes.base.node import Node
 from models.agent_config_entities import AgentSoulConfig, WorkflowNodeJobConfig
+from services.agent.prompt_mentions import extract_workflow_node_output_selectors
 
 from .ask_human_hitl import AskHumanFormBuildError, build_ask_human_pause_reason
 from .ask_human_resume import build_deferred_tool_results, resolve_ask_human_form
@@ -688,5 +689,20 @@ class DifyAgentNode(Node[DifyAgentNodeData]):
         node_id: str,
         node_data: DifyAgentNodeData,
     ) -> Mapping[str, Sequence[str]]:
-        del graph_config, node_id, node_data
-        return {}
+        """Reuse frontend workflow-marker parsing for graph variable loading.
+
+        This follows the same marker parser used by publish sync and runtime
+        request building, including reserved-prefix exclusion.
+        """
+        del graph_config
+
+        agent_task = (
+            node_data.get("agent_task") if isinstance(node_data, Mapping) else getattr(node_data, "agent_task", None)
+        )
+        if not isinstance(agent_task, str):
+            return {}
+
+        return {
+            f"{node_id}.{'.'.join(selector)}": list(selector)
+            for selector in extract_workflow_node_output_selectors(agent_task)
+        }

api/core/workflow/nodes/agent_v2/dify_tools_builder.py

@@ -0,0 +1,500 @@
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any, Literal, Protocol, cast
+
+from dify_agent.layers.dify_core_tools import DifyCoreToolConfig, DifyCoreToolProviderType, DifyCoreToolsLayerConfig
+from dify_agent.layers.dify_plugin import (
+    DifyPluginCredentialValue,
+    DifyPluginToolConfig,
+    DifyPluginToolCredentialType,
+    DifyPluginToolParameter,
+    DifyPluginToolParameterForm,
+    DifyPluginToolsLayerConfig,
+)
+from sqlalchemy import select
+from sqlalchemy.orm import Session
+
+from core.agent.entities import AgentToolEntity
+from core.app.entities.app_invoke_entities import InvokeFrom
+from core.tools.__base.tool import Tool
+from core.tools.entities.tool_entities import ToolProviderType
+from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundError
+from core.tools.tool_manager import ToolManager
+from core.tools.workflow_as_tool.provider import WorkflowToolProviderController
+from extensions.ext_database import db
+from models.agent_config_entities import AgentSoulDifyToolConfig, AgentSoulToolsConfig
+from models.provider_ids import ToolProviderID
+from models.tools import WorkflowToolProvider
+from services.tools.mcp_tools_manage_service import MCPToolManageService
+
+
+class WorkflowAgentDifyToolsBuildError(ValueError):
+    """Raised when Agent Soul tools cannot be prepared for Agent backend."""
+
+    def __init__(self, error_code: str, message: str) -> None:
+        self.error_code = error_code
+        super().__init__(message)
+
+
+class AgentToolRuntimeProvider(Protocol):
+    def get_agent_tool_runtime(
+        self,
+        tenant_id: str,
+        app_id: str,
+        agent_tool: AgentToolEntity,
+        user_id: str | None = None,
+        invoke_from: InvokeFrom = InvokeFrom.DEBUGGER,
+        variable_pool: Any | None = None,
+        allow_file_parameters: bool = False,
+        use_default_for_missing_form_parameters: bool = False,
+    ) -> Tool: ...
+
+
+class ProviderToolsLister(Protocol):
+    def __call__(
+        self,
+        *,
+        tenant_id: str,
+        provider_type: ToolProviderType,
+        provider_id: str,
+    ) -> list[str]: ...
+
+
+class MCPProviderIDResolver(Protocol):
+    def __call__(self, *, tenant_id: str, provider_id: str) -> str: ...
+
+
+@dataclass(frozen=True, slots=True)
+class WorkflowAgentToolLayers:
+    plugin_tools: DifyPluginToolsLayerConfig | None = None
+    core_tools: DifyCoreToolsLayerConfig | None = None
+
+    def exposed_tool_names(self) -> list[str]:
+        names: list[str] = []
+        if self.plugin_tools is not None:
+            names.extend(tool.name or tool.tool_name for tool in self.plugin_tools.tools)
+        if self.core_tools is not None:
+            names.extend(tool.name or tool.tool_name for tool in self.core_tools.tools)
+        return names
+
+
+class WorkflowAgentDifyToolLayersBuilder(Protocol):
+    def build_layers(
+        self,
+        *,
+        tenant_id: str,
+        app_id: str,
+        user_id: str | None,
+        tools: AgentSoulToolsConfig,
+        invoke_from: InvokeFrom,
+    ) -> WorkflowAgentToolLayers: ...
+
+
+def _list_provider_tool_names(
+    *,
+    tenant_id: str,
+    provider_type: ToolProviderType,
+    provider_id: str,
+) -> list[str]:
+    """Tool names a provider currently declares for provider-level Agent entries."""
+    match provider_type:
+        case ToolProviderType.PLUGIN:
+            plugin_provider = ToolManager.get_plugin_provider(provider_id, tenant_id)
+            return [tool.entity.identity.name for tool in plugin_provider.get_tools() or []]
+        case ToolProviderType.BUILT_IN:
+            builtin_provider = ToolManager.get_builtin_provider(provider_id, tenant_id)
+            return [tool.entity.identity.name for tool in builtin_provider.get_tools() or []]
+        case ToolProviderType.API:
+            api_provider, _ = ToolManager.get_api_provider_controller(tenant_id, provider_id)
+            return [tool.entity.identity.name for tool in api_provider.get_tools(tenant_id) or []]
+        case ToolProviderType.WORKFLOW:
+            db_provider = db.session.scalar(
+                select(WorkflowToolProvider)
+                .where(
+                    WorkflowToolProvider.id == provider_id,
+                    WorkflowToolProvider.tenant_id == tenant_id,
+                )
+                .limit(1)
+            )
+            if db_provider is None:
+                raise ToolProviderNotFoundError(f"workflow provider {provider_id} not found")
+            workflow_provider = WorkflowToolProviderController.from_db(db_provider)
+            return [tool.entity.identity.name for tool in workflow_provider.get_tools(tenant_id) or []]
+        case ToolProviderType.MCP:
+            mcp_provider = ToolManager.get_mcp_provider_controller(tenant_id, provider_id)
+            return [tool.entity.identity.name for tool in mcp_provider.get_tools() or []]
+        case _:
+            raise ToolProviderNotFoundError(f"provider type {provider_type.value} not found")
+
+
+def _resolve_mcp_provider_id(*, tenant_id: str, provider_id: str) -> str:
+    """Normalize MCP provider ids to the runtime-facing server identifier."""
+    service = MCPToolManageService(session=cast(Session, db.session))
+    try:
+        return service.get_provider_entity(provider_id, tenant_id, by_server_id=True).provider_id
+    except ValueError:
+        try:
+            return service.get_provider_entity(provider_id, tenant_id, by_server_id=False).provider_id
+        except ValueError as exc:
+            raise ToolProviderNotFoundError(f"mcp provider {provider_id} not found") from exc
+
+
+class WorkflowAgentDifyToolsBuilder:
+    """Prepare Agent Soul Dify tools for Agent backend run-layer configs.
+
+    Plugin tools keep their existing direct daemon path. Core-routed tools
+    (`builtin`/`api`/`workflow`/`mcp`) are emitted as `dify.core.tools`.
+    """
+
+    def __init__(
+        self,
+        *,
+        tool_runtime_provider: AgentToolRuntimeProvider | None = None,
+        provider_tools_lister: ProviderToolsLister | None = None,
+        mcp_provider_id_resolver: MCPProviderIDResolver | None = None,
+    ) -> None:
+        self._tool_runtime_provider = tool_runtime_provider or ToolManager
+        self._provider_tools_lister = provider_tools_lister or _list_provider_tool_names
+        self._mcp_provider_id_resolver = mcp_provider_id_resolver or _resolve_mcp_provider_id
+
+    def build_layers(
+        self,
+        *,
+        tenant_id: str,
+        app_id: str,
+        user_id: str | None,
+        tools: AgentSoulToolsConfig,
+        invoke_from: InvokeFrom,
+    ) -> WorkflowAgentToolLayers:
+        """Resolve user-selected Dify tools into direct/core Agent backend DTOs.
+
+        `invoke_from` is the real runtime caller category (DEBUGGER for a
+        Composer test run, SERVICE_API / WEB_APP for a published run). It must
+        be threaded through to `ToolManager` so credential quotas, rate limits,
+        and audit tags match the actual call site.
+        """
+        enabled_tools = [tool for tool in tools.dify_tools if tool.enabled]
+        if not enabled_tools:
+            return WorkflowAgentToolLayers()
+
+        prepared_plugin: list[DifyPluginToolConfig] = []
+        prepared_core: list[DifyCoreToolConfig] = []
+        seen_names: set[str] = set()
+
+        for tool_config in self._expand_provider_entries(tenant_id=tenant_id, enabled_tools=enabled_tools):
+            normalized_tool_config = self._normalized_tool_config(tenant_id=tenant_id, tool_config=tool_config)
+            destination = self._tool_layer_destination(normalized_tool_config)
+            exposed_name = self._exposed_tool_name(normalized_tool_config)
+            if exposed_name in seen_names:
+                raise WorkflowAgentDifyToolsBuildError(
+                    "agent_tool_name_duplicated",
+                    f"Duplicate Dify Tool name {exposed_name!r}.",
+                )
+            seen_names.add(exposed_name)
+
+            agent_tool = self._to_agent_tool_entity(normalized_tool_config)
+            tool_runtime = self._fetch_tool_runtime(
+                tenant_id=tenant_id,
+                app_id=app_id,
+                user_id=user_id,
+                agent_tool=agent_tool,
+                invoke_from=invoke_from,
+                tool_config=normalized_tool_config,
+            )
+
+            if destination == "plugin":
+                prepared_plugin.append(
+                    self._to_plugin_backend_tool_config(normalized_tool_config, tool_runtime, exposed_name)
+                )
+            else:
+                prepared_core.append(
+                    self._to_core_backend_tool_config(normalized_tool_config, tool_runtime, exposed_name)
+                )
+
+        return WorkflowAgentToolLayers(
+            plugin_tools=DifyPluginToolsLayerConfig(tools=prepared_plugin) if prepared_plugin else None,
+            core_tools=DifyCoreToolsLayerConfig(tools=prepared_core) if prepared_core else None,
+        )
+
+    def _expand_provider_entries(
+        self,
+        *,
+        tenant_id: str,
+        enabled_tools: list[AgentSoulDifyToolConfig],
+    ) -> list[AgentSoulDifyToolConfig]:
+        """Expand provider-level entries (`tool_name` omitted = all tools)."""
+        explicit_by_provider: dict[tuple[ToolProviderType, str], set[str]] = {}
+        for tool_config in enabled_tools:
+            if tool_config.tool_name is not None:
+                explicit_by_provider.setdefault(self._provider_key(tool_config), set()).add(tool_config.tool_name)
+
+        expanded: list[AgentSoulDifyToolConfig] = []
+        for tool_config in enabled_tools:
+            if tool_config.tool_name is not None:
+                expanded.append(tool_config)
+                continue
+            provider_type = ToolProviderType.value_of(tool_config.provider_type)
+            provider_id = self._provider_id(tool_config)
+            try:
+                tool_names = self._provider_declared_tool_names(
+                    tenant_id=tenant_id,
+                    provider_type=provider_type,
+                    provider_id=provider_id,
+                )
+            except ToolProviderNotFoundError as exc:
+                raise WorkflowAgentDifyToolsBuildError(
+                    "agent_tool_declaration_not_found",
+                    f"Dify Tool provider {provider_id!r} declaration not found: {exc}",
+                ) from exc
+            if not tool_names:
+                raise WorkflowAgentDifyToolsBuildError(
+                    "agent_tool_declaration_not_found",
+                    f"Dify Tool provider {provider_id!r} declares no tools.",
+                )
+            already_explicit = explicit_by_provider.get(self._provider_key(tool_config), set())
+            for tool_name in tool_names:
+                if tool_name in already_explicit:
+                    continue
+                expanded.append(tool_config.model_copy(update={"tool_name": tool_name, "runtime_parameters": {}}))
+        return expanded
+
+    def _provider_declared_tool_names(
+        self,
+        *,
+        tenant_id: str,
+        provider_type: ToolProviderType,
+        provider_id: str,
+    ) -> list[str]:
+        return self._provider_tools_lister(
+            tenant_id=tenant_id,
+            provider_type=provider_type,
+            provider_id=provider_id,
+        )
+
+    def _normalized_tool_config(
+        self,
+        *,
+        tenant_id: str,
+        tool_config: AgentSoulDifyToolConfig,
+    ) -> AgentSoulDifyToolConfig:
+        if tool_config.provider_type != ToolProviderType.MCP.value:
+            return tool_config
+        provider_id = self._mcp_provider_id_resolver(tenant_id=tenant_id, provider_id=self._provider_id(tool_config))
+        return tool_config.model_copy(update={"provider_id": provider_id, "plugin_id": None, "provider": None})
+
+    def _fetch_tool_runtime(
+        self,
+        *,
+        tenant_id: str,
+        app_id: str,
+        user_id: str | None,
+        agent_tool: AgentToolEntity,
+        invoke_from: InvokeFrom,
+        tool_config: AgentSoulDifyToolConfig,
+    ) -> Tool:
+        """Resolve the API-side `Tool` runtime and map fetch errors to stable codes."""
+        try:
+            return self._tool_runtime_provider.get_agent_tool_runtime(
+                tenant_id=tenant_id,
+                app_id=app_id,
+                agent_tool=agent_tool,
+                user_id=user_id,
+                invoke_from=invoke_from,
+                variable_pool=None,
+                allow_file_parameters=True,
+                use_default_for_missing_form_parameters=True,
+            )
+        except ToolProviderNotFoundError as exc:
+            raise WorkflowAgentDifyToolsBuildError(
+                "agent_tool_declaration_not_found",
+                f"Dify Tool {tool_config.tool_name!r} declaration not found: {exc}",
+            ) from exc
+        except ToolProviderCredentialValidationError as exc:
+            raise WorkflowAgentDifyToolsBuildError(
+                "agent_tool_credential_invalid",
+                f"Dify Tool {tool_config.tool_name!r} credential validation failed: {exc}",
+            ) from exc
+        except ValueError as exc:
+            raise WorkflowAgentDifyToolsBuildError(
+                "agent_tool_config_invalid",
+                f"Dify Tool {tool_config.tool_name!r} runtime construction failed: {exc}",
+            ) from exc
+
+    @staticmethod
+    def _to_agent_tool_entity(tool_config: AgentSoulDifyToolConfig) -> AgentToolEntity:
+        assert tool_config.tool_name is not None
+        return AgentToolEntity(
+            provider_type=ToolProviderType.value_of(tool_config.provider_type),
+            provider_id=WorkflowAgentDifyToolsBuilder._provider_id(tool_config),
+            tool_name=tool_config.tool_name,
+            tool_parameters=dict(tool_config.runtime_parameters),
+            credential_id=tool_config.credential_ref.id if tool_config.credential_ref else None,
+        )
+
+    @staticmethod
+    def _provider_id(tool_config: AgentSoulDifyToolConfig) -> str:
+        if tool_config.provider_id:
+            return tool_config.provider_id
+        assert tool_config.plugin_id is not None
+        assert tool_config.provider is not None
+        return f"{tool_config.plugin_id}/{tool_config.provider}"
+
+    @staticmethod
+    def _provider_key(tool_config: AgentSoulDifyToolConfig) -> tuple[ToolProviderType, str]:
+        return (
+            ToolProviderType.value_of(tool_config.provider_type),
+            WorkflowAgentDifyToolsBuilder._provider_id(tool_config),
+        )
+
+    @staticmethod
+    def _tool_layer_destination(tool_config: AgentSoulDifyToolConfig) -> Literal["plugin", "core"]:
+        provider_type = ToolProviderType.value_of(tool_config.provider_type)
+        if provider_type is ToolProviderType.PLUGIN:
+            return "plugin"
+        if provider_type in {
+            ToolProviderType.BUILT_IN,
+            ToolProviderType.API,
+            ToolProviderType.WORKFLOW,
+            ToolProviderType.MCP,
+        }:
+            return "core"
+        if provider_type is ToolProviderType.DATASET_RETRIEVAL:
+            raise WorkflowAgentDifyToolsBuildError(
+                "agent_tool_provider_not_supported",
+                "dataset-retrieval remains on the knowledge path and is not supported in Agent tool layers.",
+            )
+        raise WorkflowAgentDifyToolsBuildError(
+            "agent_tool_provider_not_supported",
+            f"Dify Tool provider type {provider_type.value!r} is not supported in Agent tool layers.",
+        )
+
+    @staticmethod
+    def _exposed_tool_name(tool_config: AgentSoulDifyToolConfig) -> str:
+        assert tool_config.tool_name is not None
+        return tool_config.tool_name
+
+    def _to_plugin_backend_tool_config(
+        self,
+        tool_config: AgentSoulDifyToolConfig,
+        tool_runtime: Tool,
+        exposed_name: str,
+    ) -> DifyPluginToolConfig:
+        runtime = tool_runtime.runtime
+        if runtime is None:
+            raise WorkflowAgentDifyToolsBuildError(
+                "agent_tool_config_invalid",
+                f"Dify Tool {tool_config.tool_name!r} has no runtime.",
+            )
+
+        provider_id = self._provider_id(tool_config)
+        plugin_id, provider = self._plugin_provider(tool_config, provider_id)
+        parameters = self._prepared_parameters(tool_runtime)
+        runtime_parameters = self._runtime_parameters(tool_runtime, parameters)
+        description = self._description(tool_config, tool_runtime)
+
+        return DifyPluginToolConfig(
+            plugin_id=plugin_id,
+            provider=provider,
+            tool_name=exposed_name,
+            credential_type=self._credential_type(tool_config, runtime.credentials),
+            name=exposed_name,
+            description=description,
+            credentials=self._normalize_credentials(runtime.credentials, tool_name=exposed_name),
+            runtime_parameters=runtime_parameters,
+            parameters=parameters,
+            parameters_json_schema=tool_runtime.get_llm_parameters_json_schema(),
+        )
+
+    def _to_core_backend_tool_config(
+        self,
+        tool_config: AgentSoulDifyToolConfig,
+        tool_runtime: Tool,
+        exposed_name: str,
+    ) -> DifyCoreToolConfig:
+        parameters = self._prepared_parameters(tool_runtime)
+        return DifyCoreToolConfig(
+            provider_type=cast(DifyCoreToolProviderType, tool_config.provider_type),
+            provider_id=self._provider_id(tool_config),
+            tool_name=tool_config.tool_name or exposed_name,
+            credential_id=tool_config.credential_ref.id if tool_config.credential_ref else None,
+            name=exposed_name,
+            description=self._description(tool_config, tool_runtime),
+            runtime_parameters=self._runtime_parameters(tool_runtime, parameters),
+            parameters=parameters,
+            parameters_json_schema=tool_runtime.get_llm_parameters_json_schema(),
+        )
+
+    @staticmethod
+    def _plugin_provider(tool_config: AgentSoulDifyToolConfig, provider_id: str) -> tuple[str, str]:
+        if tool_config.plugin_id and tool_config.provider:
+            return tool_config.plugin_id, tool_config.provider
+        provider_id_entity = ToolProviderID(provider_id)
+        return provider_id_entity.plugin_id, provider_id_entity.provider_name
+
+    @staticmethod
+    def _credential_type(
+        tool_config: AgentSoulDifyToolConfig,
+        credentials: Mapping[str, Any],
+    ) -> DifyPluginToolCredentialType:
+        if not credentials and tool_config.credential_type == "unauthorized":
+            return "unauthorized"
+        return tool_config.credential_type
+
+    @staticmethod
+    def _prepared_parameters(tool_runtime: Tool) -> list[DifyPluginToolParameter]:
+        return [
+            DifyPluginToolParameter.model_validate(parameter.model_dump(mode="json"))
+            for parameter in tool_runtime.get_merged_runtime_parameters()
+        ]
+
+    @staticmethod
+    def _description(tool_config: AgentSoulDifyToolConfig, tool_runtime: Tool) -> str | None:
+        description = tool_config.description
+        if description is None and tool_runtime.entity.description is not None:
+            description = tool_runtime.entity.description.llm
+        return description
+
+    @staticmethod
+    def _runtime_parameters(
+        tool_runtime: Tool,
+        parameters: list[DifyPluginToolParameter],
+    ) -> dict[str, Any]:
+        runtime = tool_runtime.runtime
+        runtime_parameters = dict(runtime.runtime_parameters if runtime is not None else {})
+        missing = [
+            parameter.name
+            for parameter in parameters
+            if parameter.form is not DifyPluginToolParameterForm.LLM
+            and parameter.required
+            and parameter.default is None
+            and parameter.name not in runtime_parameters
+        ]
+        if missing:
+            names = ", ".join(sorted(missing))
+            raise WorkflowAgentDifyToolsBuildError(
+                "agent_tool_runtime_parameter_missing",
+                f"Dify Tool {tool_runtime.entity.identity.name!r} is missing runtime parameters: {names}.",
+            )
+        return runtime_parameters
+
+    @staticmethod
+    def _normalize_credentials(
+        credentials: Mapping[str, Any],
+        *,
+        tool_name: str,
+    ) -> dict[str, DifyPluginCredentialValue]:
+        normalized: dict[str, DifyPluginCredentialValue] = {}
+        for key, value in credentials.items():
+            if isinstance(value, str | int | float | bool) or value is None:
+                normalized[key] = value
+                continue
+            raise WorkflowAgentDifyToolsBuildError(
+                "agent_tool_credential_shape_invalid",
+                (
+                    f"Dify Plugin Tool {tool_name!r} credential {key!r} has a non-scalar value "
+                    f"({type(value).__name__}); only str/int/float/bool/None are forwarded to the daemon."
+                ),
+            )
+        return normalized

api/core/workflow/nodes/agent_v2/plugin_tools_builder.py

@@ -1,334 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Mapping
-from typing import Any, Protocol
-
-from dify_agent.layers.dify_plugin import (
-    DifyPluginCredentialValue,
-    DifyPluginToolConfig,
-    DifyPluginToolCredentialType,
-    DifyPluginToolParameter,
-    DifyPluginToolParameterForm,
-    DifyPluginToolsLayerConfig,
-)
-
-from core.agent.entities import AgentToolEntity
-from core.app.entities.app_invoke_entities import InvokeFrom
-from core.tools.__base.tool import Tool
-from core.tools.entities.tool_entities import ToolProviderType
-from core.tools.errors import (
-    ToolProviderCredentialValidationError,
-    ToolProviderNotFoundError,
-)
-from core.tools.tool_manager import ToolManager
-from models.agent_config_entities import AgentSoulDifyToolConfig, AgentSoulToolsConfig
-from models.provider_ids import ToolProviderID
-
-
-class WorkflowAgentPluginToolsBuildError(ValueError):
-    """Raised when Agent Soul tools cannot be prepared for Agent backend."""
-
-    def __init__(self, error_code: str, message: str) -> None:
-        self.error_code = error_code
-        super().__init__(message)
-
-
-class AgentToolRuntimeProvider(Protocol):
-    def get_agent_tool_runtime(
-        self,
-        tenant_id: str,
-        app_id: str,
-        agent_tool: AgentToolEntity,
-        user_id: str | None = None,
-        invoke_from: InvokeFrom = InvokeFrom.DEBUGGER,
-        variable_pool: Any | None = None,
-        allow_file_parameters: bool = False,
-        use_default_for_missing_form_parameters: bool = False,
-    ) -> Tool: ...
-
-
-class ProviderToolsLister(Protocol):
-    def __call__(self, *, tenant_id: str, provider_id: str) -> list[str]: ...
-
-
-def _list_provider_tool_names(*, tenant_id: str, provider_id: str) -> list[str]:
-    """Tool names a provider currently declares (provider-level config entries)."""
-    provider = ToolManager.get_builtin_provider(provider_id, tenant_id)
-    return [tool.entity.identity.name for tool in provider.get_tools() or []]
-
-
-class WorkflowAgentPluginToolsBuilder:
-    """Prepare Agent Soul Dify Plugin Tools for the public Agent backend DTO."""
-
-    def __init__(
-        self,
-        *,
-        tool_runtime_provider: AgentToolRuntimeProvider | None = None,
-        provider_tools_lister: ProviderToolsLister | None = None,
-    ) -> None:
-        self._tool_runtime_provider = tool_runtime_provider or ToolManager
-        self._provider_tools_lister = provider_tools_lister or _list_provider_tool_names
-
-    def build(
-        self,
-        *,
-        tenant_id: str,
-        app_id: str,
-        user_id: str | None,
-        tools: AgentSoulToolsConfig,
-        invoke_from: InvokeFrom,
-    ) -> DifyPluginToolsLayerConfig | None:
-        """Resolve user-selected Dify Plugin Tools into the Agent backend DTO.
-
-        ``invoke_from`` is the *real* runtime caller category (DEBUGGER for a
-        Composer test run, SERVICE_API / WEB_APP for a published run). It must
-        be threaded through to :class:`ToolManager` so credential quotas, rate
-        limits, and audit tags match the actual call site.
-        """
-        enabled_tools = [tool for tool in tools.dify_tools if tool.enabled]
-        if not enabled_tools:
-            return None
-
-        prepared: list[DifyPluginToolConfig] = []
-        seen_names: set[str] = set()
-        for tool_config in self._expand_provider_entries(tenant_id=tenant_id, enabled_tools=enabled_tools):
-            agent_tool = self._to_agent_tool_entity(tool_config)
-            tool_runtime = self._fetch_tool_runtime(
-                tenant_id=tenant_id,
-                app_id=app_id,
-                user_id=user_id,
-                agent_tool=agent_tool,
-                invoke_from=invoke_from,
-                tool_config=tool_config,
-            )
-
-            exposed_name = self._exposed_tool_name(tool_config)
-            if exposed_name in seen_names:
-                raise WorkflowAgentPluginToolsBuildError(
-                    "agent_tool_name_duplicated",
-                    f"Duplicate Dify Plugin Tool name {exposed_name!r}.",
-                )
-            seen_names.add(exposed_name)
-
-            prepared.append(self._to_backend_tool_config(tool_config, tool_runtime, exposed_name))
-
-        return DifyPluginToolsLayerConfig(tools=prepared)
-
-    def _expand_provider_entries(
-        self,
-        *,
-        tenant_id: str,
-        enabled_tools: list[AgentSoulDifyToolConfig],
-    ) -> list[AgentSoulDifyToolConfig]:
-        """Expand provider-level entries (``tool_name`` omitted = all tools).
-
-        An explicit per-tool entry of the same provider wins over the expansion
-        (it may carry its own ``runtime_parameters``); expanded clones share the
-        provider entry's ``credential_ref`` and start with default parameters.
-        """
-        explicit_by_provider: dict[str, set[str]] = {}
-        for tool_config in enabled_tools:
-            if tool_config.tool_name is not None:
-                explicit_by_provider.setdefault(self._provider_id(tool_config), set()).add(tool_config.tool_name)
-
-        expanded: list[AgentSoulDifyToolConfig] = []
-        for tool_config in enabled_tools:
-            if tool_config.tool_name is not None:
-                expanded.append(tool_config)
-                continue
-            provider_id = self._provider_id(tool_config)
-            try:
-                tool_names = self._provider_tools_lister(tenant_id=tenant_id, provider_id=provider_id)
-            except ToolProviderNotFoundError as exc:
-                raise WorkflowAgentPluginToolsBuildError(
-                    "agent_tool_declaration_not_found",
-                    f"Dify Plugin Tool provider {provider_id!r} declaration not found: {exc}",
-                ) from exc
-            if not tool_names:
-                raise WorkflowAgentPluginToolsBuildError(
-                    "agent_tool_declaration_not_found",
-                    f"Dify Plugin Tool provider {provider_id!r} declares no tools.",
-                )
-            already_explicit = explicit_by_provider.get(provider_id, set())
-            for tool_name in tool_names:
-                if tool_name in already_explicit:
-                    continue
-                expanded.append(tool_config.model_copy(update={"tool_name": tool_name, "runtime_parameters": {}}))
-        return expanded
-
-    def _fetch_tool_runtime(
-        self,
-        *,
-        tenant_id: str,
-        app_id: str,
-        user_id: str | None,
-        agent_tool: AgentToolEntity,
-        invoke_from: InvokeFrom,
-        tool_config: AgentSoulDifyToolConfig,
-    ) -> Tool:
-        """Resolve the API-side ``Tool`` runtime, mapping fetch errors to
-        Inspector-friendly error codes so callers can render distinct UX for
-        "tool definition gone" vs "credential failed".
-        """
-        try:
-            return self._tool_runtime_provider.get_agent_tool_runtime(
-                tenant_id=tenant_id,
-                app_id=app_id,
-                agent_tool=agent_tool,
-                user_id=user_id,
-                invoke_from=invoke_from,
-                variable_pool=None,
-                allow_file_parameters=True,
-                use_default_for_missing_form_parameters=True,
-            )
-        except ToolProviderNotFoundError as exc:
-            raise WorkflowAgentPluginToolsBuildError(
-                "agent_tool_declaration_not_found",
-                f"Dify Plugin Tool {tool_config.tool_name!r} declaration not found: {exc}",
-            ) from exc
-        except ToolProviderCredentialValidationError as exc:
-            raise WorkflowAgentPluginToolsBuildError(
-                "agent_tool_credential_invalid",
-                f"Dify Plugin Tool {tool_config.tool_name!r} credential validation failed: {exc}",
-            ) from exc
-        except ValueError as exc:
-            # ToolManager raises bare ValueError when the agent tool's
-            # ``runtime`` / runtime parameters are missing. Surface it under a
-            # narrower error code than a generic "declaration not found" so
-            # frontend can render an actionable hint.
-            raise WorkflowAgentPluginToolsBuildError(
-                "agent_tool_config_invalid",
-                f"Dify Plugin Tool {tool_config.tool_name!r} runtime construction failed: {exc}",
-            ) from exc
-
-    @staticmethod
-    def _to_agent_tool_entity(tool_config: AgentSoulDifyToolConfig) -> AgentToolEntity:
-        # Provider-level entries are expanded into per-tool clones before this point.
-        assert tool_config.tool_name is not None
-        return AgentToolEntity(
-            provider_type=ToolProviderType.value_of(tool_config.provider_type),
-            provider_id=WorkflowAgentPluginToolsBuilder._provider_id(tool_config),
-            tool_name=tool_config.tool_name,
-            tool_parameters=dict(tool_config.runtime_parameters),
-            credential_id=tool_config.credential_ref.id if tool_config.credential_ref else None,
-        )
-
-    @staticmethod
-    def _provider_id(tool_config: AgentSoulDifyToolConfig) -> str:
-        if tool_config.provider_id:
-            return tool_config.provider_id
-        assert tool_config.plugin_id is not None
-        assert tool_config.provider is not None
-        return f"{tool_config.plugin_id}/{tool_config.provider}"
-
-    @staticmethod
-    def _exposed_tool_name(tool_config: AgentSoulDifyToolConfig) -> str:
-        # Stage 3.1 decision: no user rename yet. Keep the model-visible tool
-        # name aligned with the plugin declaration identity. Provider-level
-        # entries are expanded into per-tool clones before this point.
-        assert tool_config.tool_name is not None
-        return tool_config.tool_name
-
-    def _to_backend_tool_config(
-        self,
-        tool_config: AgentSoulDifyToolConfig,
-        tool_runtime: Tool,
-        exposed_name: str,
-    ) -> DifyPluginToolConfig:
-        runtime = tool_runtime.runtime
-        if runtime is None:
-            raise WorkflowAgentPluginToolsBuildError(
-                "agent_tool_config_invalid",
-                f"Dify Plugin Tool {tool_config.tool_name!r} has no runtime.",
-            )
-
-        provider_id = self._provider_id(tool_config)
-        plugin_id, provider = self._plugin_provider(tool_config, provider_id)
-        parameters = [
-            DifyPluginToolParameter.model_validate(parameter.model_dump(mode="json"))
-            for parameter in tool_runtime.get_merged_runtime_parameters()
-        ]
-        runtime_parameters = self._runtime_parameters(tool_runtime, parameters)
-        description = tool_config.description
-        if description is None and tool_runtime.entity.description is not None:
-            description = tool_runtime.entity.description.llm
-
-        return DifyPluginToolConfig(
-            plugin_id=plugin_id,
-            provider=provider,
-            tool_name=exposed_name,
-            credential_type=self._credential_type(tool_config, runtime.credentials),
-            name=exposed_name,
-            description=description,
-            credentials=self._normalize_credentials(runtime.credentials, tool_name=exposed_name),
-            runtime_parameters=runtime_parameters,
-            parameters=parameters,
-            parameters_json_schema=tool_runtime.get_llm_parameters_json_schema(),
-        )
-
-    @staticmethod
-    def _plugin_provider(tool_config: AgentSoulDifyToolConfig, provider_id: str) -> tuple[str, str]:
-        if tool_config.plugin_id and tool_config.provider:
-            return tool_config.plugin_id, tool_config.provider
-        provider_id_entity = ToolProviderID(provider_id)
-        return provider_id_entity.plugin_id, provider_id_entity.provider_name
-
-    @staticmethod
-    def _credential_type(
-        tool_config: AgentSoulDifyToolConfig,
-        credentials: Mapping[str, Any],
-    ) -> DifyPluginToolCredentialType:
-        if not credentials and tool_config.credential_type == "unauthorized":
-            return "unauthorized"
-        return tool_config.credential_type
-
-    @staticmethod
-    def _runtime_parameters(
-        tool_runtime: Tool,
-        parameters: list[DifyPluginToolParameter],
-    ) -> dict[str, Any]:
-        runtime = tool_runtime.runtime
-        runtime_parameters = dict(runtime.runtime_parameters if runtime is not None else {})
-        missing = [
-            parameter.name
-            for parameter in parameters
-            if parameter.form is not DifyPluginToolParameterForm.LLM
-            and parameter.required
-            and parameter.default is None
-            and parameter.name not in runtime_parameters
-        ]
-        if missing:
-            names = ", ".join(sorted(missing))
-            raise WorkflowAgentPluginToolsBuildError(
-                "agent_tool_runtime_parameter_missing",
-                f"Dify Plugin Tool {tool_runtime.entity.identity.name!r} is missing runtime parameters: {names}.",
-            )
-        return runtime_parameters
-
-    @staticmethod
-    def _normalize_credentials(
-        credentials: Mapping[str, Any],
-        *,
-        tool_name: str,
-    ) -> dict[str, DifyPluginCredentialValue]:
-        """Forward only scalar credential values to the Agent backend.
-
-        ``DifyPluginCredentialValue`` is ``str | int | float | bool | None``.
-        Refusing non-scalar values (lists, dicts, custom objects) is safer than
-        ``str(value)`` — stringifying a nested OAuth token blob produces a
-        Python ``repr`` that the plugin daemon cannot use, and we'd rather
-        surface a clear ``agent_tool_credential_shape_invalid`` than send junk.
-        """
-        normalized: dict[str, DifyPluginCredentialValue] = {}
-        for key, value in credentials.items():
-            if isinstance(value, str | int | float | bool) or value is None:
-                normalized[key] = value
-                continue
-            raise WorkflowAgentPluginToolsBuildError(
-                "agent_tool_credential_shape_invalid",
-                (
-                    f"Dify Plugin Tool {tool_name!r} credential {key!r} has a non-scalar value "
-                    f"({type(value).__name__}); only str/int/float/bool/None are forwarded to the daemon."
-                ),
-            )
-        return normalized