Merge branch 'main' into refactor/migrate-react-window-to-tanstack-virtual

# Conflicts:
#	web/pnpm-lock.yaml

.github/CODEOWNERS

@@ -9,9 +9,6 @@
 # CODEOWNERS file
 /.github/CODEOWNERS @laipz8200 @crazywoola
 
-# Agents
-/.agents/skills/ @hyoban
-
 # Docs
 /docs/ @crazywoola
 

api/.ruff.toml

@@ -106,10 +106,10 @@ ignore = [
     "N803", # invalid-argument-name
 ]
 "tests/*" = [
-    "F811",  # redefined-while-unused
-    "T201",  # allow print in tests,
-    "S110",  # allow ignoring exceptions in tests code (currently)
-    "PT019", # @patch-injected params look like unused fixtures
+    "F811", # redefined-while-unused
+    "T201", # allow print in tests,
+    "S110", # allow ignoring exceptions in tests code (currently)
+
 ]
 "controllers/console/explore/trial.py" = ["TID251"]
 "controllers/console/human_input_form.py" = ["TID251"]

api/README.md

@@ -122,8 +122,7 @@ These commands assume you start from the repository root.
 
    ```bash
    cd api
-   # Note: enterprise_telemetry queue is only used in Enterprise Edition
-   uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention,enterprise_telemetry
+   uv run celery -A app.celery worker -P threads -c 2 --loglevel INFO -Q dataset,priority_dataset,priority_pipeline,pipeline,mail,ops_trace,app_deletion,plugin,workflow_storage,conversation,workflow,schedule_poller,schedule_executor,triggered_workflow_dispatcher,trigger_refresh_executor,retention
    ```
 
 1. Optional: start Celery Beat (scheduled tasks, in a new terminal).

api/app_factory.py

@@ -1,45 +1,16 @@
 import logging
 import time
 
-from flask import request
 from opentelemetry.trace import get_current_span
 from opentelemetry.trace.span import INVALID_SPAN_ID, INVALID_TRACE_ID
 
 from configs import dify_config
 from contexts.wrapper import RecyclableContextVar
-from controllers.console.error import UnauthorizedAndForceLogout
 from core.logging.context import init_request_context
 from dify_app import DifyApp
-from services.enterprise.enterprise_service import EnterpriseService
-from services.feature_service import LicenseStatus
 
 logger = logging.getLogger(__name__)
 
-# Console bootstrap APIs exempt from license check.
-# Defined at module level to avoid per-request tuple construction.
-# - system-features: license status for expiry UI (GlobalPublicStoreProvider)
-# - setup: install/setup status check (AppInitializer)
-# - init: init password validation for fresh install (InitPasswordPopup)
-# - login: auto-login after setup completion (InstallForm)
-# - features: billing/plan features (ProviderContextProvider)
-# - account/profile: login check + user profile (AppContextProvider, useIsLogin)
-# - workspaces/current: workspace + model providers (AppContextProvider)
-# - version: version check (AppContextProvider)
-# - activate/check: invitation link validation (signin page)
-# Without these exemptions, the signin page triggers location.reload()
-# on unauthorized_and_force_logout, causing an infinite loop.
-_CONSOLE_EXEMPT_PREFIXES = (
-    "/console/api/system-features",
-    "/console/api/setup",
-    "/console/api/init",
-    "/console/api/login",
-    "/console/api/features",
-    "/console/api/account/profile",
-    "/console/api/workspaces/current",
-    "/console/api/version",
-    "/console/api/activate/check",
-)
-
 
 # ----------------------------
 # Application Factory Function
@@ -60,39 +31,6 @@ def create_flask_app_with_configs() -> DifyApp:
         init_request_context()
         RecyclableContextVar.increment_thread_recycles()
 
-        # Enterprise license validation for API endpoints (both console and webapp)
-        # When license expires, block all API access except bootstrap endpoints needed
-        # for the frontend to load the license expiration page without infinite reloads.
-        if dify_config.ENTERPRISE_ENABLED:
-            is_console_api = request.path.startswith("/console/api/")
-            is_webapp_api = request.path.startswith("/api/") and not is_console_api
-
-            if is_console_api or is_webapp_api:
-                if is_console_api:
-                    is_exempt = any(request.path.startswith(p) for p in _CONSOLE_EXEMPT_PREFIXES)
-                else:  # webapp API
-                    is_exempt = request.path.startswith("/api/system-features")
-
-                if not is_exempt:
-                    try:
-                        # Check license status with caching (10 min TTL)
-                        license_status = EnterpriseService.get_cached_license_status()
-                        if license_status in (LicenseStatus.INACTIVE, LicenseStatus.EXPIRED, LicenseStatus.LOST):
-                            # Cookie clearing is handled by register_external_error_handlers
-                            # in libs/external_api.py which detects the error code and calls
-                            # build_force_logout_cookie_headers(). Frontend then checks
-                            # code === 'unauthorized_and_force_logout' and calls location.reload().
-                            raise UnauthorizedAndForceLogout(
-                                f"Enterprise license is {license_status}. Please contact your administrator."
-                            )
-                    except UnauthorizedAndForceLogout:
-                        raise
-                    except Exception:
-                        # If license check fails, log but don't block the request.
-                        # This prevents service disruption if enterprise API is temporarily
-                        # unavailable.
-                        logger.exception("Failed to check enterprise license status")
-
     # add after request hook for injecting trace headers from OpenTelemetry span context
     # Only adds headers when OTEL is enabled and has valid context
     @dify_app.after_request
@@ -143,7 +81,6 @@ def initialize_extensions(app: DifyApp):
         ext_commands,
         ext_compress,
         ext_database,
-        ext_enterprise_telemetry,
         ext_fastopenapi,
         ext_forward_refs,
         ext_hosting_provider,
@@ -194,7 +131,6 @@ def initialize_extensions(app: DifyApp):
         ext_commands,
         ext_fastopenapi,
         ext_otel,
-        ext_enterprise_telemetry,
         ext_request_logging,
         ext_session_factory,
     ]

api/commands.py

@@ -30,7 +30,6 @@ from extensions.ext_redis import redis_client
 from extensions.ext_storage import storage
 from extensions.storage.opendal_storage import OpenDALStorage
 from extensions.storage.storage_type import StorageType
-from libs.db_migration_lock import DbMigrationAutoRenewLock
 from libs.helper import email as email_validate
 from libs.password import hash_password, password_pattern, valid_password
 from libs.rsa import generate_key_pair
@@ -55,8 +54,6 @@ from tasks.remove_app_and_related_data_task import delete_draft_variables_batch
 
 logger = logging.getLogger(__name__)
 
-DB_UPGRADE_LOCK_TTL_SECONDS = 60
-
 
 @click.command("reset-password", help="Reset the account password.")
 @click.option("--email", prompt=True, help="Account email to reset password for")
@@ -730,15 +727,8 @@ def create_tenant(email: str, language: str | None = None, name: str | None = No
 @click.command("upgrade-db", help="Upgrade the database")
 def upgrade_db():
     click.echo("Preparing database migration...")
-    lock = DbMigrationAutoRenewLock(
-        redis_client=redis_client,
-        name="db_upgrade_lock",
-        ttl_seconds=DB_UPGRADE_LOCK_TTL_SECONDS,
-        logger=logger,
-        log_context="db_migration",
-    )
+    lock = redis_client.lock(name="db_upgrade_lock", timeout=60)
     if lock.acquire(blocking=False):
-        migration_succeeded = False
         try:
             click.echo(click.style("Starting database migration.", fg="green"))
 
@@ -747,16 +737,12 @@ def upgrade_db():
 
             flask_migrate.upgrade()
 
-            migration_succeeded = True
             click.echo(click.style("Database migration successful!", fg="green"))
 
-        except Exception as e:
+        except Exception:
             logger.exception("Failed to execute database migration")
-            click.echo(click.style(f"Database migration failed: {e}", fg="red"))
-            raise SystemExit(1)
         finally:
-            status = "successful" if migration_succeeded else "failed"
-            lock.release_safely(status=status)
+            lock.release()
     else:
         click.echo("Database migration skipped")
 
@@ -1464,58 +1450,54 @@ def clear_orphaned_file_records(force: bool):
         all_ids_in_tables = []
         for ids_table in ids_tables:
             query = ""
-            match ids_table["type"]:
-                case "uuid":
-                    click.echo(
-                        click.style(
-                            f"- Listing file ids in column {ids_table['column']} in table {ids_table['table']}",
-                            fg="white",
-                        )
+            if ids_table["type"] == "uuid":
+                click.echo(
+                    click.style(
+                        f"- Listing file ids in column {ids_table['column']} in table {ids_table['table']}", fg="white"
                     )
-                    c = ids_table["column"]
-                    query = f"SELECT {c} FROM {ids_table['table']} WHERE {c} IS NOT NULL"
-                    with db.engine.begin() as conn:
-                        rs = conn.execute(sa.text(query))
-                    for i in rs:
-                        all_ids_in_tables.append({"table": ids_table["table"], "id": str(i[0])})
-                case "text":
-                    t = ids_table["table"]
-                    click.echo(
-                        click.style(
-                            f"- Listing file-id-like strings in column {ids_table['column']} in table {t}",
-                            fg="white",
-                        )
+                )
+                query = (
+                    f"SELECT {ids_table['column']} FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
+                )
+                with db.engine.begin() as conn:
+                    rs = conn.execute(sa.text(query))
+                for i in rs:
+                    all_ids_in_tables.append({"table": ids_table["table"], "id": str(i[0])})
+            elif ids_table["type"] == "text":
+                click.echo(
+                    click.style(
+                        f"- Listing file-id-like strings in column {ids_table['column']} in table {ids_table['table']}",
+                        fg="white",
                     )
-                    query = (
-                        f"SELECT regexp_matches({ids_table['column']}, '{guid_regexp}', 'g') AS extracted_id "
-                        f"FROM {ids_table['table']}"
+                )
+                query = (
+                    f"SELECT regexp_matches({ids_table['column']}, '{guid_regexp}', 'g') AS extracted_id "
+                    f"FROM {ids_table['table']}"
+                )
+                with db.engine.begin() as conn:
+                    rs = conn.execute(sa.text(query))
+                for i in rs:
+                    for j in i[0]:
+                        all_ids_in_tables.append({"table": ids_table["table"], "id": j})
+            elif ids_table["type"] == "json":
+                click.echo(
+                    click.style(
+                        (
+                            f"- Listing file-id-like JSON string in column {ids_table['column']} "
+                            f"in table {ids_table['table']}"
+                        ),
+                        fg="white",
                     )
-                    with db.engine.begin() as conn:
-                        rs = conn.execute(sa.text(query))
-                    for i in rs:
-                        for j in i[0]:
-                            all_ids_in_tables.append({"table": ids_table["table"], "id": j})
-                case "json":
-                    click.echo(
-                        click.style(
-                            (
-                                f"- Listing file-id-like JSON string in column {ids_table['column']} "
-                                f"in table {ids_table['table']}"
-                            ),
-                            fg="white",
-                        )
-                    )
-                    query = (
-                        f"SELECT regexp_matches({ids_table['column']}::text, '{guid_regexp}', 'g') AS extracted_id "
-                        f"FROM {ids_table['table']}"
-                    )
-                    with db.engine.begin() as conn:
-                        rs = conn.execute(sa.text(query))
-                    for i in rs:
-                        for j in i[0]:
-                            all_ids_in_tables.append({"table": ids_table["table"], "id": j})
-                case _:
-                    pass
+                )
+                query = (
+                    f"SELECT regexp_matches({ids_table['column']}::text, '{guid_regexp}', 'g') AS extracted_id "
+                    f"FROM {ids_table['table']}"
+                )
+                with db.engine.begin() as conn:
+                    rs = conn.execute(sa.text(query))
+                for i in rs:
+                    for j in i[0]:
+                        all_ids_in_tables.append({"table": ids_table["table"], "id": j})
         click.echo(click.style(f"Found {len(all_ids_in_tables)} file ids in tables.", fg="white"))
 
     except Exception as e:
@@ -1755,18 +1737,59 @@ def file_usage(
                 if src_filter != src:
                     continue
 
-        match ids_table["type"]:
-            case "uuid":
-                # Direct UUID match
-                query = (
-                    f"SELECT {ids_table['pk_column']}, {ids_table['column']} "
-                    f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
-                )
-                with db.engine.begin() as conn:
-                    rs = conn.execute(sa.text(query))
-                    for row in rs:
-                        record_id = str(row[0])
-                        ref_file_id = str(row[1])
+        if ids_table["type"] == "uuid":
+            # Direct UUID match
+            query = (
+                f"SELECT {ids_table['pk_column']}, {ids_table['column']} "
+                f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
+            )
+            with db.engine.begin() as conn:
+                rs = conn.execute(sa.text(query))
+                for row in rs:
+                    record_id = str(row[0])
+                    ref_file_id = str(row[1])
+                    if ref_file_id not in file_key_map:
+                        continue
+                    storage_key = file_key_map[ref_file_id]
+
+                    # Apply filters
+                    if file_id and ref_file_id != file_id:
+                        continue
+                    if key and not storage_key.endswith(key):
+                        continue
+
+                    # Only collect items within the requested page range
+                    if offset <= total_count < offset + limit:
+                        paginated_usages.append(
+                            {
+                                "src": f"{ids_table['table']}.{ids_table['column']}",
+                                "record_id": record_id,
+                                "file_id": ref_file_id,
+                                "key": storage_key,
+                            }
+                        )
+                    total_count += 1
+
+        elif ids_table["type"] in ("text", "json"):
+            # Extract UUIDs from text/json content
+            column_cast = f"{ids_table['column']}::text" if ids_table["type"] == "json" else ids_table["column"]
+            query = (
+                f"SELECT {ids_table['pk_column']}, {column_cast} "
+                f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
+            )
+            with db.engine.begin() as conn:
+                rs = conn.execute(sa.text(query))
+                for row in rs:
+                    record_id = str(row[0])
+                    content = str(row[1])
+
+                    # Find all UUIDs in the content
+                    import re
+
+                    uuid_pattern = re.compile(guid_regexp, re.IGNORECASE)
+                    matches = uuid_pattern.findall(content)
+
+                    for ref_file_id in matches:
                         if ref_file_id not in file_key_map:
                             continue
                         storage_key = file_key_map[ref_file_id]
@@ -1789,50 +1812,6 @@ def file_usage(
                             )
                         total_count += 1
 
-            case "text" | "json":
-                # Extract UUIDs from text/json content
-                column_cast = f"{ids_table['column']}::text" if ids_table["type"] == "json" else ids_table["column"]
-                query = (
-                    f"SELECT {ids_table['pk_column']}, {column_cast} "
-                    f"FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
-                )
-                with db.engine.begin() as conn:
-                    rs = conn.execute(sa.text(query))
-                    for row in rs:
-                        record_id = str(row[0])
-                        content = str(row[1])
-
-                        # Find all UUIDs in the content
-                        import re
-
-                        uuid_pattern = re.compile(guid_regexp, re.IGNORECASE)
-                        matches = uuid_pattern.findall(content)
-
-                        for ref_file_id in matches:
-                            if ref_file_id not in file_key_map:
-                                continue
-                            storage_key = file_key_map[ref_file_id]
-
-                            # Apply filters
-                            if file_id and ref_file_id != file_id:
-                                continue
-                            if key and not storage_key.endswith(key):
-                                continue
-
-                            # Only collect items within the requested page range
-                            if offset <= total_count < offset + limit:
-                                paginated_usages.append(
-                                    {
-                                        "src": f"{ids_table['table']}.{ids_table['column']}",
-                                        "record_id": record_id,
-                                        "file_id": ref_file_id,
-                                        "key": storage_key,
-                                    }
-                                )
-                            total_count += 1
-            case _:
-                pass
-
     # Output results
     if output_json:
         result = {

api/configs/app_config.py

@@ -8,7 +8,7 @@ from pydantic_settings import BaseSettings, PydanticBaseSettingsSource, Settings
 from libs.file_utils import search_file_upwards
 
 from .deploy import DeploymentConfig
-from .enterprise import EnterpriseFeatureConfig, EnterpriseTelemetryConfig
+from .enterprise import EnterpriseFeatureConfig
 from .extra import ExtraServiceConfig
 from .feature import FeatureConfig
 from .middleware import MiddlewareConfig
@@ -73,8 +73,6 @@ class DifyConfig(
     # Enterprise feature configs
     # **Before using, please contact business@dify.ai by email to inquire about licensing matters.**
     EnterpriseFeatureConfig,
-    # Enterprise telemetry configs
-    EnterpriseTelemetryConfig,
 ):
     model_config = SettingsConfigDict(
         # read from dotenv format config file

api/configs/enterprise/__init__.py

@@ -18,49 +18,3 @@ class EnterpriseFeatureConfig(BaseSettings):
         description="Allow customization of the enterprise logo.",
         default=False,
     )
-
-
-class EnterpriseTelemetryConfig(BaseSettings):
-    """
-    Configuration for enterprise telemetry.
-    """
-
-    ENTERPRISE_TELEMETRY_ENABLED: bool = Field(
-        description="Enable enterprise telemetry collection (also requires ENTERPRISE_ENABLED=true).",
-        default=False,
-    )
-
-    ENTERPRISE_OTLP_ENDPOINT: str = Field(
-        description="Enterprise OTEL collector endpoint.",
-        default="",
-    )
-
-    ENTERPRISE_OTLP_HEADERS: str = Field(
-        description="Auth headers for OTLP export (key=value,key2=value2).",
-        default="",
-    )
-
-    ENTERPRISE_OTLP_PROTOCOL: str = Field(
-        description="OTLP protocol: 'http' or 'grpc' (default: http).",
-        default="http",
-    )
-
-    ENTERPRISE_OTLP_API_KEY: str = Field(
-        description="Bearer token for enterprise OTLP export authentication.",
-        default="",
-    )
-
-    ENTERPRISE_INCLUDE_CONTENT: bool = Field(
-        description="Include input/output content in traces (privacy toggle).",
-        default=True,
-    )
-
-    ENTERPRISE_SERVICE_NAME: str = Field(
-        description="Service name for OTEL resource.",
-        default="dify",
-    )
-
-    ENTERPRISE_OTEL_SAMPLING_RATE: float = Field(
-        description="Sampling rate for enterprise traces (0.0 to 1.0, default 1.0 = 100%).",
-        default=1.0,
-    )

api/controllers/console/app/annotation.py

@@ -1,11 +1,10 @@
 from typing import Any, Literal
 
 from flask import abort, make_response, request
-from flask_restx import Resource
-from pydantic import BaseModel, Field, TypeAdapter, field_validator
+from flask_restx import Resource, fields, marshal, marshal_with
+from pydantic import BaseModel, Field, field_validator
 
 from controllers.common.errors import NoFileUploadedError, TooManyFilesError
-from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import (
     account_initialization_required,
@@ -17,11 +16,9 @@ from controllers.console.wraps import (
 )
 from extensions.ext_redis import redis_client
 from fields.annotation_fields import (
-    Annotation,
-    AnnotationExportList,
-    AnnotationHitHistory,
-    AnnotationHitHistoryList,
-    AnnotationList,
+    annotation_fields,
+    annotation_hit_history_fields,
+    build_annotation_model,
 )
 from libs.helper import uuid_value
 from libs.login import login_required
@@ -92,14 +89,6 @@ reg(CreateAnnotationPayload)
 reg(UpdateAnnotationPayload)
 reg(AnnotationReplyStatusQuery)
 reg(AnnotationFilePayload)
-register_schema_models(
-    console_ns,
-    Annotation,
-    AnnotationList,
-    AnnotationExportList,
-    AnnotationHitHistory,
-    AnnotationHitHistoryList,
-)
 
 
 @console_ns.route("/apps/<uuid:app_id>/annotation-reply/<string:action>")
@@ -118,11 +107,10 @@ class AnnotationReplyActionApi(Resource):
     def post(self, app_id, action: Literal["enable", "disable"]):
         app_id = str(app_id)
         args = AnnotationReplyPayload.model_validate(console_ns.payload)
-        match action:
-            case "enable":
-                result = AppAnnotationService.enable_app_annotation(args.model_dump(), app_id)
-            case "disable":
-                result = AppAnnotationService.disable_app_annotation(app_id)
+        if action == "enable":
+            result = AppAnnotationService.enable_app_annotation(args.model_dump(), app_id)
+        elif action == "disable":
+            result = AppAnnotationService.disable_app_annotation(app_id)
         return result, 200
 
 
@@ -213,33 +201,33 @@ class AnnotationApi(Resource):
 
         app_id = str(app_id)
         annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id(app_id, page, limit, keyword)
-        annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
-        response = AnnotationList(
-            data=annotation_models,
-            has_more=len(annotation_list) == limit,
-            limit=limit,
-            total=total,
-            page=page,
-        )
-        return response.model_dump(mode="json"), 200
+        response = {
+            "data": marshal(annotation_list, annotation_fields),
+            "has_more": len(annotation_list) == limit,
+            "limit": limit,
+            "total": total,
+            "page": page,
+        }
+        return response, 200
 
     @console_ns.doc("create_annotation")
     @console_ns.doc(description="Create a new annotation for an app")
     @console_ns.doc(params={"app_id": "Application ID"})
     @console_ns.expect(console_ns.models[CreateAnnotationPayload.__name__])
-    @console_ns.response(201, "Annotation created successfully", console_ns.models[Annotation.__name__])
+    @console_ns.response(201, "Annotation created successfully", build_annotation_model(console_ns))
     @console_ns.response(403, "Insufficient permissions")
     @setup_required
     @login_required
     @account_initialization_required
     @cloud_edition_billing_resource_check("annotation")
+    @marshal_with(annotation_fields)
     @edit_permission_required
     def post(self, app_id):
         app_id = str(app_id)
         args = CreateAnnotationPayload.model_validate(console_ns.payload)
         data = args.model_dump(exclude_none=True)
         annotation = AppAnnotationService.up_insert_app_annotation_from_message(data, app_id)
-        return Annotation.model_validate(annotation, from_attributes=True).model_dump(mode="json")
+        return annotation
 
     @setup_required
     @login_required
@@ -276,7 +264,7 @@ class AnnotationExportApi(Resource):
     @console_ns.response(
         200,
         "Annotations exported successfully",
-        console_ns.models[AnnotationExportList.__name__],
+        console_ns.model("AnnotationList", {"data": fields.List(fields.Nested(build_annotation_model(console_ns)))}),
     )
     @console_ns.response(403, "Insufficient permissions")
     @setup_required
@@ -286,8 +274,7 @@ class AnnotationExportApi(Resource):
     def get(self, app_id):
         app_id = str(app_id)
         annotation_list = AppAnnotationService.export_annotation_list_by_app_id(app_id)
-        annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
-        response_data = AnnotationExportList(data=annotation_models).model_dump(mode="json")
+        response_data = {"data": marshal(annotation_list, annotation_fields)}
 
         # Create response with secure headers for CSV export
         response = make_response(response_data, 200)
@@ -302,7 +289,7 @@ class AnnotationUpdateDeleteApi(Resource):
     @console_ns.doc("update_delete_annotation")
     @console_ns.doc(description="Update or delete an annotation")
     @console_ns.doc(params={"app_id": "Application ID", "annotation_id": "Annotation ID"})
-    @console_ns.response(200, "Annotation updated successfully", console_ns.models[Annotation.__name__])
+    @console_ns.response(200, "Annotation updated successfully", build_annotation_model(console_ns))
     @console_ns.response(204, "Annotation deleted successfully")
     @console_ns.response(403, "Insufficient permissions")
     @console_ns.expect(console_ns.models[UpdateAnnotationPayload.__name__])
@@ -311,6 +298,7 @@ class AnnotationUpdateDeleteApi(Resource):
     @account_initialization_required
     @cloud_edition_billing_resource_check("annotation")
     @edit_permission_required
+    @marshal_with(annotation_fields)
     def post(self, app_id, annotation_id):
         app_id = str(app_id)
         annotation_id = str(annotation_id)
@@ -318,7 +306,7 @@ class AnnotationUpdateDeleteApi(Resource):
         annotation = AppAnnotationService.update_app_annotation_directly(
             args.model_dump(exclude_none=True), app_id, annotation_id
         )
-        return Annotation.model_validate(annotation, from_attributes=True).model_dump(mode="json")
+        return annotation
 
     @setup_required
     @login_required
@@ -426,7 +414,14 @@ class AnnotationHitHistoryListApi(Resource):
     @console_ns.response(
         200,
         "Hit histories retrieved successfully",
-        console_ns.models[AnnotationHitHistoryList.__name__],
+        console_ns.model(
+            "AnnotationHitHistoryList",
+            {
+                "data": fields.List(
+                    fields.Nested(console_ns.model("AnnotationHitHistoryItem", annotation_hit_history_fields))
+                )
+            },
+        ),
     )
     @console_ns.response(403, "Insufficient permissions")
     @setup_required
@@ -441,14 +436,11 @@ class AnnotationHitHistoryListApi(Resource):
         annotation_hit_history_list, total = AppAnnotationService.get_annotation_hit_histories(
             app_id, annotation_id, page, limit
         )
-        history_models = TypeAdapter(list[AnnotationHitHistory]).validate_python(
-            annotation_hit_history_list, from_attributes=True
-        )
-        response = AnnotationHitHistoryList(
-            data=history_models,
-            has_more=len(annotation_hit_history_list) == limit,
-            limit=limit,
-            total=total,
-            page=page,
-        )
-        return response.model_dump(mode="json")
+        response = {
+            "data": marshal(annotation_hit_history_list, annotation_hit_history_fields),
+            "has_more": len(annotation_hit_history_list) == limit,
+            "limit": limit,
+            "total": total,
+            "page": page,
+        }
+        return response

api/controllers/console/app/app.py

@@ -1,4 +1,3 @@
-import logging
 import uuid
 from datetime import datetime
 from typing import Any, Literal, TypeAlias
@@ -55,8 +54,6 @@ ALLOW_CREATE_APP_MODES = ["chat", "agent-chat", "advanced-chat", "workflow", "co
 
 register_enum_models(console_ns, IconType)
 
-_logger = logging.getLogger(__name__)
-
 
 class AppListQuery(BaseModel):
     page: int = Field(default=1, ge=1, le=99999, description="Page number (1-99999)")
@@ -502,7 +499,6 @@ class AppListApi(Resource):
                     select(Workflow).where(
                         Workflow.version == Workflow.VERSION_DRAFT,
                         Workflow.app_id.in_(workflow_capable_app_ids),
-                        Workflow.tenant_id == current_tenant_id,
                     )
                 )
                 .scalars()
@@ -514,14 +510,12 @@ class AppListApi(Resource):
                 NodeType.TRIGGER_PLUGIN,
             }
             for workflow in draft_workflows:
-                node_id = None
                 try:
-                    for node_id, node_data in workflow.walk_nodes():
+                    for _, node_data in workflow.walk_nodes():
                         if node_data.get("type") in trigger_node_types:
                             draft_trigger_app_ids.add(str(workflow.app_id))
                             break
                 except Exception:
-                    _logger.exception("error while walking nodes, workflow_id=%s, node_id=%s", workflow.id, node_id)
                     continue
 
         for app in app_pagination.items:
@@ -660,19 +654,6 @@ class AppCopyApi(Resource):
             )
             session.commit()
 
-            # Inherit web app permission from original app
-            if result.app_id and FeatureService.get_system_features().webapp_auth.enabled:
-                try:
-                    # Get the original app's access mode
-                    original_settings = EnterpriseService.WebAppAuth.get_app_access_mode_by_id(app_model.id)
-                    access_mode = original_settings.access_mode
-                except Exception:
-                    # If original app has no settings (old app), default to public to match fallback behavior
-                    access_mode = "public"
-
-                # Apply the same access mode to the copied app
-                EnterpriseService.WebAppAuth.update_app_access_mode(result.app_id, access_mode)
-
             stmt = select(App).where(App.id == result.app_id)
             app = session.scalar(stmt)
 

api/controllers/console/app/audio.py

@@ -6,7 +6,6 @@ from pydantic import BaseModel, Field
 from werkzeug.exceptions import InternalServerError
 
 import services
-from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.app.error import (
     AppUnavailableError,
@@ -34,6 +33,7 @@ from services.errors.audio import (
 )
 
 logger = logging.getLogger(__name__)
+DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
 
 
 class TextToSpeechPayload(BaseModel):
@@ -47,11 +47,13 @@ class TextToSpeechVoiceQuery(BaseModel):
     language: str = Field(..., description="Language code")
 
 
-class AudioTranscriptResponse(BaseModel):
-    text: str = Field(description="Transcribed text from audio")
-
-
-register_schema_models(console_ns, AudioTranscriptResponse, TextToSpeechPayload, TextToSpeechVoiceQuery)
+console_ns.schema_model(
+    TextToSpeechPayload.__name__, TextToSpeechPayload.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0)
+)
+console_ns.schema_model(
+    TextToSpeechVoiceQuery.__name__,
+    TextToSpeechVoiceQuery.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0),
+)
 
 
 @console_ns.route("/apps/<uuid:app_id>/audio-to-text")
@@ -62,7 +64,7 @@ class ChatMessageAudioApi(Resource):
     @console_ns.response(
         200,
         "Audio transcription successful",
-        console_ns.models[AudioTranscriptResponse.__name__],
+        console_ns.model("AudioTranscriptResponse", {"text": fields.String(description="Transcribed text from audio")}),
     )
     @console_ns.response(400, "Bad request - No audio uploaded or unsupported type")
     @console_ns.response(413, "Audio file too large")

api/controllers/console/app/conversation.py

@@ -508,19 +508,16 @@ class ChatConversationApi(Resource):
                 case "created_at" | "-created_at" | _:
                     query = query.where(Conversation.created_at <= end_datetime_utc)
 
-        match args.annotation_status:
-            case "annotated":
-                query = query.options(joinedload(Conversation.message_annotations)).join(  # type: ignore
-                    MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
-                )
-            case "not_annotated":
-                query = (
-                    query.outerjoin(MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id)
-                    .group_by(Conversation.id)
-                    .having(func.count(MessageAnnotation.id) == 0)
-                )
-            case "all":
-                pass
+        if args.annotation_status == "annotated":
+            query = query.options(joinedload(Conversation.message_annotations)).join(  # type: ignore
+                MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id
+            )
+        elif args.annotation_status == "not_annotated":
+            query = (
+                query.outerjoin(MessageAnnotation, MessageAnnotation.conversation_id == Conversation.id)
+                .group_by(Conversation.id)
+                .having(func.count(MessageAnnotation.id) == 0)
+            )
 
         if app_model.mode == AppMode.ADVANCED_CHAT:
             query = query.where(Conversation.invoke_from != InvokeFrom.DEBUGGER)

api/controllers/console/app/generator.py

@@ -35,7 +35,6 @@ class InstructionGeneratePayload(BaseModel):
     instruction: str = Field(..., description="Instruction for generation")
     model_config_data: ModelConfig = Field(..., alias="model_config", description="Model configuration")
     ideal_output: str = Field(default="", description="Expected ideal output")
-    app_id: str | None = Field(default=None, description="App ID for prompt generation tracing")
 
 
 class InstructionTemplatePayload(BaseModel):
@@ -67,17 +66,10 @@ class RuleGenerateApi(Resource):
     @account_initialization_required
     def post(self):
         args = RuleGeneratePayload.model_validate(console_ns.payload)
-        account, current_tenant_id = current_account_with_tenant()
+        _, current_tenant_id = current_account_with_tenant()
 
         try:
-            rules = LLMGenerator.generate_rule_config(
-                tenant_id=current_tenant_id,
-                instruction=args.instruction,
-                model_config=args.model_config_data,
-                no_variable=args.no_variable,
-                user_id=account.id,
-                app_id=args.app_id,
-            )
+            rules = LLMGenerator.generate_rule_config(tenant_id=current_tenant_id, args=args)
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
         except QuotaExceededError:
@@ -103,16 +95,12 @@ class RuleCodeGenerateApi(Resource):
     @account_initialization_required
     def post(self):
         args = RuleCodeGeneratePayload.model_validate(console_ns.payload)
-        account, current_tenant_id = current_account_with_tenant()
+        _, current_tenant_id = current_account_with_tenant()
 
         try:
             code_result = LLMGenerator.generate_code(
                 tenant_id=current_tenant_id,
-                instruction=args.instruction,
-                model_config=args.model_config_data,
-                code_language=args.code_language,
-                user_id=account.id,
-                app_id=args.app_id,
+                args=args,
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -139,15 +127,12 @@ class RuleStructuredOutputGenerateApi(Resource):
     @account_initialization_required
     def post(self):
         args = RuleStructuredOutputPayload.model_validate(console_ns.payload)
-        account, current_tenant_id = current_account_with_tenant()
+        _, current_tenant_id = current_account_with_tenant()
 
         try:
             structured_output = LLMGenerator.generate_structured_output(
                 tenant_id=current_tenant_id,
-                instruction=args.instruction,
-                model_config=args.model_config_data,
-                user_id=account.id,
-                app_id=args.app_id,
+                args=args,
             )
         except ProviderTokenNotInitError as ex:
             raise ProviderNotInitializeError(ex.description)
@@ -174,14 +159,14 @@ class InstructionGenerateApi(Resource):
     @account_initialization_required
     def post(self):
         args = InstructionGeneratePayload.model_validate(console_ns.payload)
-        account, current_tenant_id = current_account_with_tenant()
-        app_id = args.app_id or args.flow_id
+        _, current_tenant_id = current_account_with_tenant()
         providers: list[type[CodeNodeProvider]] = [Python3CodeProvider, JavascriptCodeProvider]
         code_provider: type[CodeNodeProvider] | None = next(
             (p for p in providers if p.is_accept_language(args.language)), None
         )
         code_template = code_provider.get_default_code() if code_provider else ""
         try:
+            # Generate from nothing for a workflow node
             if (args.current in (code_template, "")) and args.node_id != "":
                 app = db.session.query(App).where(App.id == args.flow_id).first()
                 if not app:
@@ -198,33 +183,33 @@ class InstructionGenerateApi(Resource):
                     case "llm":
                         return LLMGenerator.generate_rule_config(
                             current_tenant_id,
-                            instruction=args.instruction,
-                            model_config=args.model_config_data,
-                            no_variable=True,
-                            user_id=account.id,
-                            app_id=app_id,
+                            args=RuleGeneratePayload(
+                                instruction=args.instruction,
+                                model_config=args.model_config_data,
+                                no_variable=True,
+                            ),
                         )
                     case "agent":
                         return LLMGenerator.generate_rule_config(
                             current_tenant_id,
-                            instruction=args.instruction,
-                            model_config=args.model_config_data,
-                            no_variable=True,
-                            user_id=account.id,
-                            app_id=app_id,
+                            args=RuleGeneratePayload(
+                                instruction=args.instruction,
+                                model_config=args.model_config_data,
+                                no_variable=True,
+                            ),
                         )
                     case "code":
                         return LLMGenerator.generate_code(
                             tenant_id=current_tenant_id,
-                            instruction=args.instruction,
-                            model_config=args.model_config_data,
-                            code_language=args.language,
-                            user_id=account.id,
-                            app_id=app_id,
+                            args=RuleCodeGeneratePayload(
+                                instruction=args.instruction,
+                                model_config=args.model_config_data,
+                                code_language=args.language,
+                            ),
                         )
                     case _:
                         return {"error": f"invalid node type: {node_type}"}
-            if args.node_id == "" and args.current != "":
+            if args.node_id == "" and args.current != "":  # For legacy app without a workflow
                 return LLMGenerator.instruction_modify_legacy(
                     tenant_id=current_tenant_id,
                     flow_id=args.flow_id,
@@ -232,10 +217,8 @@ class InstructionGenerateApi(Resource):
                     instruction=args.instruction,
                     model_config=args.model_config_data,
                     ideal_output=args.ideal_output,
-                    user_id=account.id,
-                    app_id=app_id,
                 )
-            if args.node_id != "" and args.current != "":
+            if args.node_id != "" and args.current != "":  # For workflow node
                 return LLMGenerator.instruction_modify_workflow(
                     tenant_id=current_tenant_id,
                     flow_id=args.flow_id,
@@ -245,8 +228,6 @@ class InstructionGenerateApi(Resource):
                     model_config=args.model_config_data,
                     ideal_output=args.ideal_output,
                     workflow_service=WorkflowService(),
-                    user_id=account.id,
-                    app_id=app_id,
                 )
             return {"error": "incompatible parameters"}, 400
         except ProviderTokenNotInitError as ex:

api/controllers/console/app/message.py

@@ -7,7 +7,6 @@ from pydantic import BaseModel, Field, field_validator
 from sqlalchemy import exists, select
 from werkzeug.exceptions import InternalServerError, NotFound
 
-from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.app.error import (
     CompletionRequestError,
@@ -36,6 +35,7 @@ from services.errors.message import MessageNotExistsError, SuggestedQuestionsAft
 from services.message_service import MessageService
 
 logger = logging.getLogger(__name__)
+DEFAULT_REF_TEMPLATE_SWAGGER_2_0 = "#/definitions/{model}"
 
 
 class ChatMessagesQuery(BaseModel):
@@ -90,22 +90,13 @@ class FeedbackExportQuery(BaseModel):
         raise ValueError("has_comment must be a boolean value")
 
 
-class AnnotationCountResponse(BaseModel):
-    count: int = Field(description="Number of annotations")
+def reg(cls: type[BaseModel]):
+    console_ns.schema_model(cls.__name__, cls.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0))
 
 
-class SuggestedQuestionsResponse(BaseModel):
-    data: list[str] = Field(description="Suggested question")
-
-
-register_schema_models(
-    console_ns,
-    ChatMessagesQuery,
-    MessageFeedbackPayload,
-    FeedbackExportQuery,
-    AnnotationCountResponse,
-    SuggestedQuestionsResponse,
-)
+reg(ChatMessagesQuery)
+reg(MessageFeedbackPayload)
+reg(FeedbackExportQuery)
 
 # Register models for flask_restx to avoid dict type issues in Swagger
 # Register in dependency order: base models first, then dependent models
@@ -240,7 +231,7 @@ class ChatMessageListApi(Resource):
     @marshal_with(message_infinite_scroll_pagination_model)
     @edit_permission_required
     def get(self, app_model):
-        args = ChatMessagesQuery.model_validate(request.args.to_dict())
+        args = ChatMessagesQuery.model_validate(request.args.to_dict(flat=True))  # type: ignore
 
         conversation = (
             db.session.query(Conversation)
@@ -365,7 +356,7 @@ class MessageAnnotationCountApi(Resource):
     @console_ns.response(
         200,
         "Annotation count retrieved successfully",
-        console_ns.models[AnnotationCountResponse.__name__],
+        console_ns.model("AnnotationCountResponse", {"count": fields.Integer(description="Number of annotations")}),
     )
     @get_app_model
     @setup_required
@@ -385,7 +376,9 @@ class MessageSuggestedQuestionApi(Resource):
     @console_ns.response(
         200,
         "Suggested questions retrieved successfully",
-        console_ns.models[SuggestedQuestionsResponse.__name__],
+        console_ns.model(
+            "SuggestedQuestionsResponse", {"data": fields.List(fields.String(description="Suggested question"))}
+        ),
     )
     @console_ns.response(404, "Message or conversation not found")
     @setup_required
@@ -435,7 +428,7 @@ class MessageFeedbackExportApi(Resource):
     @login_required
     @account_initialization_required
     def get(self, app_model):
-        args = FeedbackExportQuery.model_validate(request.args.to_dict())
+        args = FeedbackExportQuery.model_validate(request.args.to_dict(flat=True))  # type: ignore
 
         # Import the service function
         from services.feedback_service import FeedbackService

api/controllers/console/app/ops_trace.py

@@ -1,7 +1,6 @@
 from typing import Any
 
 from flask import request
-from flask_login import current_user
 from flask_restx import Resource, fields
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import BadRequest
@@ -78,10 +77,7 @@ class TraceAppConfigApi(Resource):
 
         try:
             result = OpsService.create_tracing_app_config(
-                app_id=app_id,
-                tracing_provider=args.tracing_provider,
-                tracing_config=args.tracing_config,
-                account_id=current_user.id,
+                app_id=app_id, tracing_provider=args.tracing_provider, tracing_config=args.tracing_config
             )
             if not result:
                 raise TracingConfigIsExist()
@@ -106,10 +102,7 @@ class TraceAppConfigApi(Resource):
 
         try:
             result = OpsService.update_tracing_app_config(
-                app_id=app_id,
-                tracing_provider=args.tracing_provider,
-                tracing_config=args.tracing_config,
-                account_id=current_user.id,
+                app_id=app_id, tracing_provider=args.tracing_provider, tracing_config=args.tracing_config
             )
             if not result:
                 raise TracingConfigNotExist()
@@ -131,9 +124,7 @@ class TraceAppConfigApi(Resource):
         args = TraceProviderQuery.model_validate(request.args.to_dict(flat=True))  # type: ignore
 
         try:
-            result = OpsService.delete_tracing_app_config(
-                app_id=app_id, tracing_provider=args.tracing_provider, account_id=current_user.id
-            )
+            result = OpsService.delete_tracing_app_config(app_id=app_id, tracing_provider=args.tracing_provider)
             if not result:
                 raise TracingConfigNotExist()
             return {"result": "success"}, 204

api/controllers/console/auth/data_source_oauth.py

@@ -2,11 +2,9 @@ import logging
 
 import httpx
 from flask import current_app, redirect, request
-from flask_restx import Resource
-from pydantic import BaseModel, Field
+from flask_restx import Resource, fields
 
 from configs import dify_config
-from controllers.common.schema import register_schema_models
 from libs.login import login_required
 from libs.oauth_data_source import NotionOAuth
 
@@ -16,26 +14,6 @@ from ..wraps import account_initialization_required, is_admin_or_owner_required,
 logger = logging.getLogger(__name__)
 
 
-class OAuthDataSourceResponse(BaseModel):
-    data: str = Field(description="Authorization URL or 'internal' for internal setup")
-
-
-class OAuthDataSourceBindingResponse(BaseModel):
-    result: str = Field(description="Operation result")
-
-
-class OAuthDataSourceSyncResponse(BaseModel):
-    result: str = Field(description="Operation result")
-
-
-register_schema_models(
-    console_ns,
-    OAuthDataSourceResponse,
-    OAuthDataSourceBindingResponse,
-    OAuthDataSourceSyncResponse,
-)
-
-
 def get_oauth_providers():
     with current_app.app_context():
         notion_oauth = NotionOAuth(
@@ -56,7 +34,10 @@ class OAuthDataSource(Resource):
     @console_ns.response(
         200,
         "Authorization URL or internal setup success",
-        console_ns.models[OAuthDataSourceResponse.__name__],
+        console_ns.model(
+            "OAuthDataSourceResponse",
+            {"data": fields.Raw(description="Authorization URL or 'internal' for internal setup")},
+        ),
     )
     @console_ns.response(400, "Invalid provider")
     @console_ns.response(403, "Admin privileges required")
@@ -120,7 +101,7 @@ class OAuthDataSourceBinding(Resource):
     @console_ns.response(
         200,
         "Data source binding success",
-        console_ns.models[OAuthDataSourceBindingResponse.__name__],
+        console_ns.model("OAuthDataSourceBindingResponse", {"result": fields.String(description="Operation result")}),
     )
     @console_ns.response(400, "Invalid provider or code")
     def get(self, provider: str):
@@ -152,7 +133,7 @@ class OAuthDataSourceSync(Resource):
     @console_ns.response(
         200,
         "Data source sync success",
-        console_ns.models[OAuthDataSourceSyncResponse.__name__],
+        console_ns.model("OAuthDataSourceSyncResponse", {"result": fields.String(description="Operation result")}),
     )
     @console_ns.response(400, "Invalid provider or sync failed")
     @setup_required

api/controllers/console/auth/forgot_password.py

@@ -2,11 +2,10 @@ import base64
 import secrets
 
 from flask import request
-from flask_restx import Resource
+from flask_restx import Resource, fields
 from pydantic import BaseModel, Field, field_validator
 from sqlalchemy.orm import Session
 
-from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.auth.error import (
     EmailCodeError,
@@ -49,31 +48,8 @@ class ForgotPasswordResetPayload(BaseModel):
         return valid_password(value)
 
 
-class ForgotPasswordEmailResponse(BaseModel):
-    result: str = Field(description="Operation result")
-    data: str | None = Field(default=None, description="Reset token")
-    code: str | None = Field(default=None, description="Error code if account not found")
-
-
-class ForgotPasswordCheckResponse(BaseModel):
-    is_valid: bool = Field(description="Whether code is valid")
-    email: EmailStr = Field(description="Email address")
-    token: str = Field(description="New reset token")
-
-
-class ForgotPasswordResetResponse(BaseModel):
-    result: str = Field(description="Operation result")
-
-
-register_schema_models(
-    console_ns,
-    ForgotPasswordSendPayload,
-    ForgotPasswordCheckPayload,
-    ForgotPasswordResetPayload,
-    ForgotPasswordEmailResponse,
-    ForgotPasswordCheckResponse,
-    ForgotPasswordResetResponse,
-)
+for model in (ForgotPasswordSendPayload, ForgotPasswordCheckPayload, ForgotPasswordResetPayload):
+    console_ns.schema_model(model.__name__, model.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0))
 
 
 @console_ns.route("/forgot-password")
@@ -84,7 +60,14 @@ class ForgotPasswordSendEmailApi(Resource):
     @console_ns.response(
         200,
         "Email sent successfully",
-        console_ns.models[ForgotPasswordEmailResponse.__name__],
+        console_ns.model(
+            "ForgotPasswordEmailResponse",
+            {
+                "result": fields.String(description="Operation result"),
+                "data": fields.String(description="Reset token"),
+                "code": fields.String(description="Error code if account not found"),
+            },
+        ),
     )
     @console_ns.response(400, "Invalid email or rate limit exceeded")
     @setup_required
@@ -123,7 +106,14 @@ class ForgotPasswordCheckApi(Resource):
     @console_ns.response(
         200,
         "Code verified successfully",
-        console_ns.models[ForgotPasswordCheckResponse.__name__],
+        console_ns.model(
+            "ForgotPasswordCheckResponse",
+            {
+                "is_valid": fields.Boolean(description="Whether code is valid"),
+                "email": fields.String(description="Email address"),
+                "token": fields.String(description="New reset token"),
+            },
+        ),
     )
     @console_ns.response(400, "Invalid code or token")
     @setup_required
@@ -173,7 +163,7 @@ class ForgotPasswordResetApi(Resource):
     @console_ns.response(
         200,
         "Password reset successfully",
-        console_ns.models[ForgotPasswordResetResponse.__name__],
+        console_ns.model("ForgotPasswordResetResponse", {"result": fields.String(description="Operation result")}),
     )
     @console_ns.response(400, "Invalid token or password mismatch")
     @setup_required

api/controllers/console/auth/oauth_server.py

@@ -155,43 +155,43 @@ class OAuthServerUserTokenApi(Resource):
             grant_type = OAuthGrantType(payload.grant_type)
         except ValueError:
             raise BadRequest("invalid grant_type")
-        match grant_type:
-            case OAuthGrantType.AUTHORIZATION_CODE:
-                if not payload.code:
-                    raise BadRequest("code is required")
 
-                if payload.client_secret != oauth_provider_app.client_secret:
-                    raise BadRequest("client_secret is invalid")
+        if grant_type == OAuthGrantType.AUTHORIZATION_CODE:
+            if not payload.code:
+                raise BadRequest("code is required")
 
-                if payload.redirect_uri not in oauth_provider_app.redirect_uris:
-                    raise BadRequest("redirect_uri is invalid")
+            if payload.client_secret != oauth_provider_app.client_secret:
+                raise BadRequest("client_secret is invalid")
 
-                access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
-                    grant_type, code=payload.code, client_id=oauth_provider_app.client_id
-                )
-                return jsonable_encoder(
-                    {
-                        "access_token": access_token,
-                        "token_type": "Bearer",
-                        "expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
-                        "refresh_token": refresh_token,
-                    }
-                )
-            case OAuthGrantType.REFRESH_TOKEN:
-                if not payload.refresh_token:
-                    raise BadRequest("refresh_token is required")
+            if payload.redirect_uri not in oauth_provider_app.redirect_uris:
+                raise BadRequest("redirect_uri is invalid")
 
-                access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
-                    grant_type, refresh_token=payload.refresh_token, client_id=oauth_provider_app.client_id
-                )
-                return jsonable_encoder(
-                    {
-                        "access_token": access_token,
-                        "token_type": "Bearer",
-                        "expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
-                        "refresh_token": refresh_token,
-                    }
-                )
+            access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
+                grant_type, code=payload.code, client_id=oauth_provider_app.client_id
+            )
+            return jsonable_encoder(
+                {
+                    "access_token": access_token,
+                    "token_type": "Bearer",
+                    "expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
+                    "refresh_token": refresh_token,
+                }
+            )
+        elif grant_type == OAuthGrantType.REFRESH_TOKEN:
+            if not payload.refresh_token:
+                raise BadRequest("refresh_token is required")
+
+            access_token, refresh_token = OAuthServerService.sign_oauth_access_token(
+                grant_type, refresh_token=payload.refresh_token, client_id=oauth_provider_app.client_id
+            )
+            return jsonable_encoder(
+                {
+                    "access_token": access_token,
+                    "token_type": "Bearer",
+                    "expires_in": OAUTH_ACCESS_TOKEN_EXPIRES_IN,
+                    "refresh_token": refresh_token,
+                }
+            )
 
 
 @console_ns.route("/oauth/provider/account")

api/controllers/console/datasets/data_source.py

@@ -1,6 +1,6 @@
 import json
 from collections.abc import Generator
-from typing import Any, Literal, cast
+from typing import Any, cast
 
 from flask import request
 from flask_restx import Resource, fields, marshal_with
@@ -157,8 +157,9 @@ class DataSourceApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    def patch(self, binding_id, action: Literal["enable", "disable"]):
+    def patch(self, binding_id, action):
         binding_id = str(binding_id)
+        action = str(action)
         with Session(db.engine) as session:
             data_source_binding = session.execute(
                 select(DataSourceOauthBinding).filter_by(id=binding_id)
@@ -166,24 +167,23 @@ class DataSourceApi(Resource):
         if data_source_binding is None:
             raise NotFound("Data source binding not found.")
         # enable binding
-        match action:
-            case "enable":
-                if data_source_binding.disabled:
-                    data_source_binding.disabled = False
-                    data_source_binding.updated_at = naive_utc_now()
-                    db.session.add(data_source_binding)
-                    db.session.commit()
-                else:
-                    raise ValueError("Data source is not disabled.")
-            # disable binding
-            case "disable":
-                if not data_source_binding.disabled:
-                    data_source_binding.disabled = True
-                    data_source_binding.updated_at = naive_utc_now()
-                    db.session.add(data_source_binding)
-                    db.session.commit()
-                else:
-                    raise ValueError("Data source is disabled.")
+        if action == "enable":
+            if data_source_binding.disabled:
+                data_source_binding.disabled = False
+                data_source_binding.updated_at = naive_utc_now()
+                db.session.add(data_source_binding)
+                db.session.commit()
+            else:
+                raise ValueError("Data source is not disabled.")
+        # disable binding
+        if action == "disable":
+            if not data_source_binding.disabled:
+                data_source_binding.disabled = True
+                data_source_binding.updated_at = naive_utc_now()
+                db.session.add(data_source_binding)
+                db.session.commit()
+            else:
+                raise ValueError("Data source is disabled.")
         return {"result": "success"}, 200
 
 

api/controllers/console/datasets/datasets_document.py

@@ -576,62 +576,63 @@ class DocumentBatchIndexingEstimateApi(DocumentResource):
             if document.indexing_status in {"completed", "error"}:
                 raise DocumentAlreadyFinishedError()
             data_source_info = document.data_source_info_dict
-            match document.data_source_type:
-                case "upload_file":
-                    if not data_source_info:
-                        continue
-                    file_id = data_source_info["upload_file_id"]
-                    file_detail = (
-                        db.session.query(UploadFile)
-                        .where(UploadFile.tenant_id == current_tenant_id, UploadFile.id == file_id)
-                        .first()
-                    )
 
-                    if file_detail is None:
-                        raise NotFound("File not found.")
+            if document.data_source_type == "upload_file":
+                if not data_source_info:
+                    continue
+                file_id = data_source_info["upload_file_id"]
+                file_detail = (
+                    db.session.query(UploadFile)
+                    .where(UploadFile.tenant_id == current_tenant_id, UploadFile.id == file_id)
+                    .first()
+                )
 
-                    extract_setting = ExtractSetting(
-                        datasource_type=DatasourceType.FILE, upload_file=file_detail, document_model=document.doc_form
-                    )
-                    extract_settings.append(extract_setting)
-                case "notion_import":
-                    if not data_source_info:
-                        continue
-                    extract_setting = ExtractSetting(
-                        datasource_type=DatasourceType.NOTION,
-                        notion_info=NotionInfo.model_validate(
-                            {
-                                "credential_id": data_source_info.get("credential_id"),
-                                "notion_workspace_id": data_source_info["notion_workspace_id"],
-                                "notion_obj_id": data_source_info["notion_page_id"],
-                                "notion_page_type": data_source_info["type"],
-                                "tenant_id": current_tenant_id,
-                            }
-                        ),
-                        document_model=document.doc_form,
-                    )
-                    extract_settings.append(extract_setting)
-                case "website_crawl":
-                    if not data_source_info:
-                        continue
-                    extract_setting = ExtractSetting(
-                        datasource_type=DatasourceType.WEBSITE,
-                        website_info=WebsiteInfo.model_validate(
-                            {
-                                "provider": data_source_info["provider"],
-                                "job_id": data_source_info["job_id"],
-                                "url": data_source_info["url"],
-                                "tenant_id": current_tenant_id,
-                                "mode": data_source_info["mode"],
-                                "only_main_content": data_source_info["only_main_content"],
-                            }
-                        ),
-                        document_model=document.doc_form,
-                    )
-                    extract_settings.append(extract_setting)
+                if file_detail is None:
+                    raise NotFound("File not found.")
 
-                case _:
-                    raise ValueError("Data source type not support")
+                extract_setting = ExtractSetting(
+                    datasource_type=DatasourceType.FILE, upload_file=file_detail, document_model=document.doc_form
+                )
+                extract_settings.append(extract_setting)
+
+            elif document.data_source_type == "notion_import":
+                if not data_source_info:
+                    continue
+                extract_setting = ExtractSetting(
+                    datasource_type=DatasourceType.NOTION,
+                    notion_info=NotionInfo.model_validate(
+                        {
+                            "credential_id": data_source_info.get("credential_id"),
+                            "notion_workspace_id": data_source_info["notion_workspace_id"],
+                            "notion_obj_id": data_source_info["notion_page_id"],
+                            "notion_page_type": data_source_info["type"],
+                            "tenant_id": current_tenant_id,
+                        }
+                    ),
+                    document_model=document.doc_form,
+                )
+                extract_settings.append(extract_setting)
+            elif document.data_source_type == "website_crawl":
+                if not data_source_info:
+                    continue
+                extract_setting = ExtractSetting(
+                    datasource_type=DatasourceType.WEBSITE,
+                    website_info=WebsiteInfo.model_validate(
+                        {
+                            "provider": data_source_info["provider"],
+                            "job_id": data_source_info["job_id"],
+                            "url": data_source_info["url"],
+                            "tenant_id": current_tenant_id,
+                            "mode": data_source_info["mode"],
+                            "only_main_content": data_source_info["only_main_content"],
+                        }
+                    ),
+                    document_model=document.doc_form,
+                )
+                extract_settings.append(extract_setting)
+
+            else:
+                raise ValueError("Data source type not support")
             indexing_runner = IndexingRunner()
             try:
                 response = indexing_runner.indexing_estimate(
@@ -953,24 +954,23 @@ class DocumentProcessingApi(DocumentResource):
         if not current_user.is_dataset_editor:
             raise Forbidden()
 
-        match action:
-            case "pause":
-                if document.indexing_status != "indexing":
-                    raise InvalidActionError("Document not in indexing state.")
+        if action == "pause":
+            if document.indexing_status != "indexing":
+                raise InvalidActionError("Document not in indexing state.")
 
-                document.paused_by = current_user.id
-                document.paused_at = naive_utc_now()
-                document.is_paused = True
-                db.session.commit()
+            document.paused_by = current_user.id
+            document.paused_at = naive_utc_now()
+            document.is_paused = True
+            db.session.commit()
 
-            case "resume":
-                if document.indexing_status not in {"paused", "error"}:
-                    raise InvalidActionError("Document not in paused or error state.")
+        elif action == "resume":
+            if document.indexing_status not in {"paused", "error"}:
+                raise InvalidActionError("Document not in paused or error state.")
 
-                document.paused_by = None
-                document.paused_at = None
-                document.is_paused = False
-                db.session.commit()
+            document.paused_by = None
+            document.paused_at = None
+            document.is_paused = False
+            db.session.commit()
 
         return {"result": "success"}, 200
 
@@ -1339,18 +1339,6 @@ class DocumentGenerateSummaryApi(Resource):
             missing_ids = set(document_list) - found_ids
             raise NotFound(f"Some documents not found: {list(missing_ids)}")
 
-        # Update need_summary to True for documents that don't have it set
-        # This handles the case where documents were created when summary_index_setting was disabled
-        documents_to_update = [doc for doc in documents if not doc.need_summary and doc.doc_form != "qa_model"]
-
-        if documents_to_update:
-            document_ids_to_update = [str(doc.id) for doc in documents_to_update]
-            DocumentService.update_documents_need_summary(
-                dataset_id=dataset_id,
-                document_ids=document_ids_to_update,
-                need_summary=True,
-            )
-
         # Dispatch async tasks for each document
         for document in documents:
             # Skip qa_model documents as they don't generate summaries

api/controllers/console/datasets/metadata.py

@@ -126,11 +126,10 @@ class DatasetMetadataBuiltInFieldActionApi(Resource):
             raise NotFound("Dataset not found.")
         DatasetService.check_dataset_permission(dataset, current_user)
 
-        match action:
-            case "enable":
-                MetadataService.enable_built_in_field(dataset)
-            case "disable":
-                MetadataService.disable_built_in_field(dataset)
+        if action == "enable":
+            MetadataService.enable_built_in_field(dataset)
+        elif action == "disable":
+            MetadataService.disable_built_in_field(dataset)
         return {"result": "success"}, 200
 
 

api/controllers/console/explore/trial.py

@@ -9,7 +9,7 @@ import services
 from controllers.common.fields import Parameters as ParametersResponse
 from controllers.common.fields import Site as SiteResponse
 from controllers.common.schema import get_or_create_model
-from controllers.console import api
+from controllers.console import api, console_ns
 from controllers.console.app.error import (
     AppUnavailableError,
     AudioTooLargeError,
@@ -51,7 +51,7 @@ from fields.app_fields import (
     tag_fields,
 )
 from fields.dataset_fields import dataset_fields
-from fields.member_fields import simple_account_fields
+from fields.member_fields import build_simple_account_model
 from fields.workflow_fields import (
     conversation_variable_fields,
     pipeline_variable_fields,
@@ -103,7 +103,7 @@ app_detail_fields_with_site_copy["tags"] = fields.List(fields.Nested(tag_model))
 app_detail_fields_with_site_copy["site"] = fields.Nested(site_model)
 app_detail_with_site_model = get_or_create_model("TrialAppDetailWithSite", app_detail_fields_with_site_copy)
 
-simple_account_model = get_or_create_model("SimpleAccount", simple_account_fields)
+simple_account_model = build_simple_account_model(console_ns)
 conversation_variable_model = get_or_create_model("TrialConversationVariable", conversation_variable_fields)
 pipeline_variable_model = get_or_create_model("TrialPipelineVariable", pipeline_variable_fields)
 

api/controllers/console/feature.py

@@ -1,60 +1,58 @@
-from flask_restx import Resource, fields
+from pydantic import BaseModel, Field
 from werkzeug.exceptions import Unauthorized
 
+from controllers.fastopenapi import console_router
 from libs.login import current_account_with_tenant, current_user, login_required
-from services.feature_service import FeatureService
+from services.feature_service import FeatureModel, FeatureService, SystemFeatureModel
 
-from . import console_ns
 from .wraps import account_initialization_required, cloud_utm_record, setup_required
 
 
-@console_ns.route("/features")
-class FeatureApi(Resource):
-    @console_ns.doc("get_tenant_features")
-    @console_ns.doc(description="Get feature configuration for current tenant")
-    @console_ns.response(
-        200,
-        "Success",
-        console_ns.model("FeatureResponse", {"features": fields.Raw(description="Feature configuration object")}),
-    )
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @cloud_utm_record
-    def get(self):
-        """Get feature configuration for current tenant"""
-        _, current_tenant_id = current_account_with_tenant()
-
-        return FeatureService.get_features(current_tenant_id).model_dump()
+class FeatureResponse(BaseModel):
+    features: FeatureModel = Field(description="Feature configuration object")
 
 
-@console_ns.route("/system-features")
-class SystemFeatureApi(Resource):
-    @console_ns.doc("get_system_features")
-    @console_ns.doc(description="Get system-wide feature configuration")
-    @console_ns.response(
-        200,
-        "Success",
-        console_ns.model(
-            "SystemFeatureResponse", {"features": fields.Raw(description="System feature configuration object")}
-        ),
-    )
-    def get(self):
-        """Get system-wide feature configuration
+class SystemFeatureResponse(BaseModel):
+    features: SystemFeatureModel = Field(description="System feature configuration object")
 
-        NOTE: This endpoint is unauthenticated by design, as it provides system features
-        data required for dashboard initialization.
 
-        Authentication would create circular dependency (can't login without dashboard loading).
+@console_router.get(
+    "/features",
+    response_model=FeatureResponse,
+    tags=["console"],
+)
+@setup_required
+@login_required
+@account_initialization_required
+@cloud_utm_record
+def get_tenant_features() -> FeatureResponse:
+    """Get feature configuration for current tenant."""
+    _, current_tenant_id = current_account_with_tenant()
 
-        Only non-sensitive configuration data should be returned by this endpoint.
-        """
-        # NOTE(QuantumGhost): ideally we should access `current_user.is_authenticated`
-        # without a try-catch. However, due to the implementation of user loader (the `load_user_from_request`
-        # in api/extensions/ext_login.py), accessing `current_user.is_authenticated` will
-        # raise `Unauthorized` exception if authentication token is not provided.
-        try:
-            is_authenticated = current_user.is_authenticated
-        except Unauthorized:
-            is_authenticated = False
-        return FeatureService.get_system_features(is_authenticated=is_authenticated).model_dump()
+    return FeatureResponse(features=FeatureService.get_features(current_tenant_id))
+
+
+@console_router.get(
+    "/system-features",
+    response_model=SystemFeatureResponse,
+    tags=["console"],
+)
+def get_system_features() -> SystemFeatureResponse:
+    """Get system-wide feature configuration
+
+    NOTE: This endpoint is unauthenticated by design, as it provides system features
+    data required for dashboard initialization.
+
+    Authentication would create circular dependency (can't login without dashboard loading).
+
+    Only non-sensitive configuration data should be returned by this endpoint.
+    """
+    # NOTE(QuantumGhost): ideally we should access `current_user.is_authenticated`
+    # without a try-catch. However, due to the implementation of user loader (the `load_user_from_request`
+    # in api/extensions/ext_login.py), accessing `current_user.is_authenticated` will
+    # raise `Unauthorized` exception if authentication token is not provided.
+    try:
+        is_authenticated = current_user.is_authenticated
+    except Unauthorized:
+        is_authenticated = False
+    return SystemFeatureResponse(features=FeatureService.get_system_features(is_authenticated=is_authenticated))

api/controllers/console/tag/tags.py

@@ -1,27 +1,14 @@
 from typing import Literal
+from uuid import UUID
 
-from flask import request
-from flask_restx import Namespace, Resource, fields, marshal_with
 from pydantic import BaseModel, Field
 from werkzeug.exceptions import Forbidden
 
-from controllers.common.schema import register_schema_models
-from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, edit_permission_required, setup_required
+from controllers.fastopenapi import console_router
 from libs.login import current_account_with_tenant, login_required
 from services.tag_service import TagService
 
-dataset_tag_fields = {
-    "id": fields.String,
-    "name": fields.String,
-    "type": fields.String,
-    "binding_count": fields.String,
-}
-
-
-def build_dataset_tag_fields(api_or_ns: Namespace):
-    return api_or_ns.model("DataSetTag", dataset_tag_fields)
-
 
 class TagBasePayload(BaseModel):
     name: str = Field(description="Tag name", min_length=1, max_length=50)
@@ -45,115 +32,129 @@ class TagListQueryParam(BaseModel):
     keyword: str | None = Field(None, description="Search keyword")
 
 
-register_schema_models(
-    console_ns,
-    TagBasePayload,
-    TagBindingPayload,
-    TagBindingRemovePayload,
-    TagListQueryParam,
+class TagResponse(BaseModel):
+    id: str = Field(description="Tag ID")
+    name: str = Field(description="Tag name")
+    type: str = Field(description="Tag type")
+    binding_count: int = Field(description="Number of bindings")
+
+
+class TagBindingResult(BaseModel):
+    result: Literal["success"] = Field(description="Operation result", examples=["success"])
+
+
+@console_router.get(
+    "/tags",
+    response_model=list[TagResponse],
+    tags=["console"],
 )
+@setup_required
+@login_required
+@account_initialization_required
+def list_tags(query: TagListQueryParam) -> list[TagResponse]:
+    _, current_tenant_id = current_account_with_tenant()
+    tags = TagService.get_tags(query.type, current_tenant_id, query.keyword)
+
+    return [
+        TagResponse(
+            id=tag.id,
+            name=tag.name,
+            type=tag.type,
+            binding_count=int(tag.binding_count),
+        )
+        for tag in tags
+    ]
 
 
-@console_ns.route("/tags")
-class TagListApi(Resource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @console_ns.doc(
-        params={"type": 'Tag type filter. Can be "knowledge" or "app".', "keyword": "Search keyword for tag name."}
-    )
-    @marshal_with(dataset_tag_fields)
-    def get(self):
-        _, current_tenant_id = current_account_with_tenant()
-        raw_args = request.args.to_dict()
-        param = TagListQueryParam.model_validate(raw_args)
-        tags = TagService.get_tags(param.type, current_tenant_id, param.keyword)
+@console_router.post(
+    "/tags",
+    response_model=TagResponse,
+    tags=["console"],
+)
+@setup_required
+@login_required
+@account_initialization_required
+def create_tag(payload: TagBasePayload) -> TagResponse:
+    current_user, _ = current_account_with_tenant()
+    # The role of the current user in the tag table must be admin, owner, or editor
+    if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+        raise Forbidden()
 
-        return tags, 200
+    tag = TagService.save_tags(payload.model_dump())
 
-    @console_ns.expect(console_ns.models[TagBasePayload.__name__])
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self):
-        current_user, _ = current_account_with_tenant()
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
-            raise Forbidden()
-
-        payload = TagBasePayload.model_validate(console_ns.payload or {})
-        tag = TagService.save_tags(payload.model_dump())
-
-        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0}
-
-        return response, 200
+    return TagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count=0)
 
 
-@console_ns.route("/tags/<uuid:tag_id>")
-class TagUpdateDeleteApi(Resource):
-    @console_ns.expect(console_ns.models[TagBasePayload.__name__])
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def patch(self, tag_id):
-        current_user, _ = current_account_with_tenant()
-        tag_id = str(tag_id)
-        # The role of the current user in the ta table must be admin, owner, or editor
-        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
-            raise Forbidden()
+@console_router.patch(
+    "/tags/<uuid:tag_id>",
+    response_model=TagResponse,
+    tags=["console"],
+)
+@setup_required
+@login_required
+@account_initialization_required
+def update_tag(tag_id: UUID, payload: TagBasePayload) -> TagResponse:
+    current_user, _ = current_account_with_tenant()
+    tag_id_str = str(tag_id)
+    # The role of the current user in the ta table must be admin, owner, or editor
+    if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+        raise Forbidden()
 
-        payload = TagBasePayload.model_validate(console_ns.payload or {})
-        tag = TagService.update_tags(payload.model_dump(), tag_id)
+    tag = TagService.update_tags(payload.model_dump(), tag_id_str)
 
-        binding_count = TagService.get_tag_binding_count(tag_id)
+    binding_count = TagService.get_tag_binding_count(tag_id_str)
 
-        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count}
-
-        return response, 200
-
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @edit_permission_required
-    def delete(self, tag_id):
-        tag_id = str(tag_id)
-
-        TagService.delete_tag(tag_id)
-
-        return 204
+    return TagResponse(id=tag.id, name=tag.name, type=tag.type, binding_count=binding_count)
 
 
-@console_ns.route("/tag-bindings/create")
-class TagBindingCreateApi(Resource):
-    @console_ns.expect(console_ns.models[TagBindingPayload.__name__])
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self):
-        current_user, _ = current_account_with_tenant()
-        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
-        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
-            raise Forbidden()
+@console_router.delete(
+    "/tags/<uuid:tag_id>",
+    tags=["console"],
+    status_code=204,
+)
+@setup_required
+@login_required
+@account_initialization_required
+@edit_permission_required
+def delete_tag(tag_id: UUID) -> None:
+    tag_id_str = str(tag_id)
 
-        payload = TagBindingPayload.model_validate(console_ns.payload or {})
-        TagService.save_tag_binding(payload.model_dump())
-
-        return {"result": "success"}, 200
+    TagService.delete_tag(tag_id_str)
 
 
-@console_ns.route("/tag-bindings/remove")
-class TagBindingDeleteApi(Resource):
-    @console_ns.expect(console_ns.models[TagBindingRemovePayload.__name__])
-    @setup_required
-    @login_required
-    @account_initialization_required
-    def post(self):
-        current_user, _ = current_account_with_tenant()
-        # The role of the current user in the ta table must be admin, owner, editor, or dataset_operator
-        if not (current_user.has_edit_permission or current_user.is_dataset_editor):
-            raise Forbidden()
+@console_router.post(
+    "/tag-bindings/create",
+    response_model=TagBindingResult,
+    tags=["console"],
+)
+@setup_required
+@login_required
+@account_initialization_required
+def create_tag_binding(payload: TagBindingPayload) -> TagBindingResult:
+    current_user, _ = current_account_with_tenant()
+    # The role of the current user in the tag table must be admin, owner, editor, or dataset_operator
+    if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+        raise Forbidden()
 
-        payload = TagBindingRemovePayload.model_validate(console_ns.payload or {})
-        TagService.delete_tag_binding(payload.model_dump())
+    TagService.save_tag_binding(payload.model_dump())
 
-        return {"result": "success"}, 200
+    return TagBindingResult(result="success")
+
+
+@console_router.post(
+    "/tag-bindings/remove",
+    response_model=TagBindingResult,
+    tags=["console"],
+)
+@setup_required
+@login_required
+@account_initialization_required
+def delete_tag_binding(payload: TagBindingRemovePayload) -> TagBindingResult:
+    current_user, _ = current_account_with_tenant()
+    # The role of the current user in the tag table must be admin, owner, editor, or dataset_operator
+    if not (current_user.has_edit_permission or current_user.is_dataset_editor):
+        raise Forbidden()
+
+    TagService.delete_tag_binding(payload.model_dump())
+
+    return TagBindingResult(result="success")

api/controllers/console/workspace/account.py

@@ -12,7 +12,6 @@ from sqlalchemy.orm import Session
 
 from configs import dify_config
 from constants.languages import supported_language
-from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.auth.error import (
     EmailAlreadyInUseError,
@@ -38,7 +37,7 @@ from controllers.console.wraps import (
     setup_required,
 )
 from extensions.ext_database import db
-from fields.member_fields import Account as AccountResponse
+from fields.member_fields import account_fields
 from libs.datetime_utils import naive_utc_now
 from libs.helper import EmailStr, TimestampField, extract_remote_ip, timezone
 from libs.login import current_account_with_tenant, login_required
@@ -171,12 +170,6 @@ reg(ChangeEmailSendPayload)
 reg(ChangeEmailValidityPayload)
 reg(ChangeEmailResetPayload)
 reg(CheckEmailUniquePayload)
-register_schema_models(console_ns, AccountResponse)
-
-
-def _serialize_account(account) -> dict:
-    return AccountResponse.model_validate(account, from_attributes=True).model_dump(mode="json")
-
 
 integrate_fields = {
     "provider": fields.String,
@@ -243,11 +236,11 @@ class AccountProfileApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
+    @marshal_with(account_fields)
     @enterprise_license_required
     def get(self):
         current_user, _ = current_account_with_tenant()
-        return _serialize_account(current_user)
+        return current_user
 
 
 @console_ns.route("/account/name")
@@ -256,14 +249,14 @@ class AccountNameApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
+    @marshal_with(account_fields)
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
         args = AccountNamePayload.model_validate(payload)
         updated_account = AccountService.update_account(current_user, name=args.name)
 
-        return _serialize_account(updated_account)
+        return updated_account
 
 
 @console_ns.route("/account/avatar")
@@ -272,7 +265,7 @@ class AccountAvatarApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
+    @marshal_with(account_fields)
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -280,7 +273,7 @@ class AccountAvatarApi(Resource):
 
         updated_account = AccountService.update_account(current_user, avatar=args.avatar)
 
-        return _serialize_account(updated_account)
+        return updated_account
 
 
 @console_ns.route("/account/interface-language")
@@ -289,7 +282,7 @@ class AccountInterfaceLanguageApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
+    @marshal_with(account_fields)
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -297,7 +290,7 @@ class AccountInterfaceLanguageApi(Resource):
 
         updated_account = AccountService.update_account(current_user, interface_language=args.interface_language)
 
-        return _serialize_account(updated_account)
+        return updated_account
 
 
 @console_ns.route("/account/interface-theme")
@@ -306,7 +299,7 @@ class AccountInterfaceThemeApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
+    @marshal_with(account_fields)
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -314,7 +307,7 @@ class AccountInterfaceThemeApi(Resource):
 
         updated_account = AccountService.update_account(current_user, interface_theme=args.interface_theme)
 
-        return _serialize_account(updated_account)
+        return updated_account
 
 
 @console_ns.route("/account/timezone")
@@ -323,7 +316,7 @@ class AccountTimezoneApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
+    @marshal_with(account_fields)
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -331,7 +324,7 @@ class AccountTimezoneApi(Resource):
 
         updated_account = AccountService.update_account(current_user, timezone=args.timezone)
 
-        return _serialize_account(updated_account)
+        return updated_account
 
 
 @console_ns.route("/account/password")
@@ -340,7 +333,7 @@ class AccountPasswordApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
+    @marshal_with(account_fields)
     def post(self):
         current_user, _ = current_account_with_tenant()
         payload = console_ns.payload or {}
@@ -351,7 +344,7 @@ class AccountPasswordApi(Resource):
         except ServiceCurrentPasswordIncorrectError:
             raise CurrentPasswordIncorrectError()
 
-        return _serialize_account(current_user)
+        return {"result": "success"}
 
 
 @console_ns.route("/account/integrates")
@@ -627,7 +620,7 @@ class ChangeEmailResetApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountResponse.__name__])
+    @marshal_with(account_fields)
     def post(self):
         payload = console_ns.payload or {}
         args = ChangeEmailResetPayload.model_validate(payload)
@@ -656,7 +649,7 @@ class ChangeEmailResetApi(Resource):
             email=normalized_new_email,
         )
 
-        return _serialize_account(updated_account)
+        return updated_account
 
 
 @console_ns.route("/account/change-email/check-email-unique")

api/controllers/console/workspace/endpoint.py

@@ -1,10 +1,9 @@
 from typing import Any
 
 from flask import request
-from flask_restx import Resource
+from flask_restx import Resource, fields
 from pydantic import BaseModel, Field
 
-from controllers.common.schema import register_schema_models
 from controllers.console import console_ns
 from controllers.console.wraps import account_initialization_required, is_admin_or_owner_required, setup_required
 from core.model_runtime.utils.encoders import jsonable_encoder
@@ -39,53 +38,15 @@ class EndpointListForPluginQuery(EndpointListQuery):
     plugin_id: str
 
 
-class EndpointCreateResponse(BaseModel):
-    success: bool = Field(description="Operation success")
-
-
-class EndpointListResponse(BaseModel):
-    endpoints: list[dict[str, Any]] = Field(description="Endpoint information")
-
-
-class PluginEndpointListResponse(BaseModel):
-    endpoints: list[dict[str, Any]] = Field(description="Endpoint information")
-
-
-class EndpointDeleteResponse(BaseModel):
-    success: bool = Field(description="Operation success")
-
-
-class EndpointUpdateResponse(BaseModel):
-    success: bool = Field(description="Operation success")
-
-
-class EndpointEnableResponse(BaseModel):
-    success: bool = Field(description="Operation success")
-
-
-class EndpointDisableResponse(BaseModel):
-    success: bool = Field(description="Operation success")
-
-
 def reg(cls: type[BaseModel]):
     console_ns.schema_model(cls.__name__, cls.model_json_schema(ref_template=DEFAULT_REF_TEMPLATE_SWAGGER_2_0))
 
 
-register_schema_models(
-    console_ns,
-    EndpointCreatePayload,
-    EndpointIdPayload,
-    EndpointUpdatePayload,
-    EndpointListQuery,
-    EndpointListForPluginQuery,
-    EndpointCreateResponse,
-    EndpointListResponse,
-    PluginEndpointListResponse,
-    EndpointDeleteResponse,
-    EndpointUpdateResponse,
-    EndpointEnableResponse,
-    EndpointDisableResponse,
-)
+reg(EndpointCreatePayload)
+reg(EndpointIdPayload)
+reg(EndpointUpdatePayload)
+reg(EndpointListQuery)
+reg(EndpointListForPluginQuery)
 
 
 @console_ns.route("/workspaces/current/endpoints/create")
@@ -96,7 +57,7 @@ class EndpointCreateApi(Resource):
     @console_ns.response(
         200,
         "Endpoint created successfully",
-        console_ns.models[EndpointCreateResponse.__name__],
+        console_ns.model("EndpointCreateResponse", {"success": fields.Boolean(description="Operation success")}),
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required
@@ -130,7 +91,9 @@ class EndpointListApi(Resource):
     @console_ns.response(
         200,
         "Success",
-        console_ns.models[EndpointListResponse.__name__],
+        console_ns.model(
+            "EndpointListResponse", {"endpoints": fields.List(fields.Raw(description="Endpoint information"))}
+        ),
     )
     @setup_required
     @login_required
@@ -163,7 +126,9 @@ class EndpointListForSinglePluginApi(Resource):
     @console_ns.response(
         200,
         "Success",
-        console_ns.models[PluginEndpointListResponse.__name__],
+        console_ns.model(
+            "PluginEndpointListResponse", {"endpoints": fields.List(fields.Raw(description="Endpoint information"))}
+        ),
     )
     @setup_required
     @login_required
@@ -198,7 +163,7 @@ class EndpointDeleteApi(Resource):
     @console_ns.response(
         200,
         "Endpoint deleted successfully",
-        console_ns.models[EndpointDeleteResponse.__name__],
+        console_ns.model("EndpointDeleteResponse", {"success": fields.Boolean(description="Operation success")}),
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required
@@ -225,7 +190,7 @@ class EndpointUpdateApi(Resource):
     @console_ns.response(
         200,
         "Endpoint updated successfully",
-        console_ns.models[EndpointUpdateResponse.__name__],
+        console_ns.model("EndpointUpdateResponse", {"success": fields.Boolean(description="Operation success")}),
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required
@@ -256,7 +221,7 @@ class EndpointEnableApi(Resource):
     @console_ns.response(
         200,
         "Endpoint enabled successfully",
-        console_ns.models[EndpointEnableResponse.__name__],
+        console_ns.model("EndpointEnableResponse", {"success": fields.Boolean(description="Operation success")}),
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required
@@ -283,7 +248,7 @@ class EndpointDisableApi(Resource):
     @console_ns.response(
         200,
         "Endpoint disabled successfully",
-        console_ns.models[EndpointDisableResponse.__name__],
+        console_ns.model("EndpointDisableResponse", {"success": fields.Boolean(description="Operation success")}),
     )
     @console_ns.response(403, "Admin privileges required")
     @setup_required

api/controllers/console/workspace/members.py

@@ -1,12 +1,12 @@
 from urllib import parse
 
 from flask import abort, request
-from flask_restx import Resource
-from pydantic import BaseModel, Field, TypeAdapter
+from flask_restx import Resource, fields, marshal_with
+from pydantic import BaseModel, Field
 
 import services
 from configs import dify_config
-from controllers.common.schema import register_enum_models, register_schema_models
+from controllers.common.schema import get_or_create_model, register_enum_models
 from controllers.console import console_ns
 from controllers.console.auth.error import (
     CannotTransferOwnerToSelfError,
@@ -25,7 +25,7 @@ from controllers.console.wraps import (
     setup_required,
 )
 from extensions.ext_database import db
-from fields.member_fields import AccountWithRole, AccountWithRoleList
+from fields.member_fields import account_with_role_fields, account_with_role_list_fields
 from libs.helper import extract_remote_ip
 from libs.login import current_account_with_tenant, login_required
 from models.account import Account, TenantAccountRole
@@ -69,7 +69,12 @@ reg(OwnerTransferEmailPayload)
 reg(OwnerTransferCheckPayload)
 reg(OwnerTransferPayload)
 register_enum_models(console_ns, TenantAccountRole)
-register_schema_models(console_ns, AccountWithRole, AccountWithRoleList)
+
+account_with_role_model = get_or_create_model("AccountWithRole", account_with_role_fields)
+
+account_with_role_list_fields_copy = account_with_role_list_fields.copy()
+account_with_role_list_fields_copy["accounts"] = fields.List(fields.Nested(account_with_role_model))
+account_with_role_list_model = get_or_create_model("AccountWithRoleList", account_with_role_list_fields_copy)
 
 
 @console_ns.route("/workspaces/current/members")
@@ -79,15 +84,13 @@ class MemberListApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountWithRoleList.__name__])
+    @marshal_with(account_with_role_list_model)
     def get(self):
         current_user, _ = current_account_with_tenant()
         if not current_user.current_tenant:
             raise ValueError("No current tenant")
         members = TenantService.get_tenant_members(current_user.current_tenant)
-        member_models = TypeAdapter(list[AccountWithRole]).validate_python(members, from_attributes=True)
-        response = AccountWithRoleList(accounts=member_models)
-        return response.model_dump(mode="json"), 200
+        return {"result": "success", "accounts": members}, 200
 
 
 @console_ns.route("/workspaces/current/members/invite-email")
@@ -232,15 +235,13 @@ class DatasetOperatorMemberListApi(Resource):
     @setup_required
     @login_required
     @account_initialization_required
-    @console_ns.response(200, "Success", console_ns.models[AccountWithRoleList.__name__])
+    @marshal_with(account_with_role_list_model)
     def get(self):
         current_user, _ = current_account_with_tenant()
         if not current_user.current_tenant:
             raise ValueError("No current tenant")
         members = TenantService.get_dataset_operator_members(current_user.current_tenant)
-        member_models = TypeAdapter(list[AccountWithRole]).validate_python(members, from_attributes=True)
-        response = AccountWithRoleList(accounts=member_models)
-        return response.model_dump(mode="json"), 200
+        return {"result": "success", "accounts": members}, 200
 
 
 @console_ns.route("/workspaces/current/members/send-owner-transfer-confirm-email")

api/controllers/console/workspace/tool_providers.py

@@ -878,11 +878,7 @@ class ToolBuiltinProviderSetDefaultApi(Resource):
         current_user, current_tenant_id = current_account_with_tenant()
         payload = BuiltinProviderDefaultCredentialPayload.model_validate(console_ns.payload or {})
         return BuiltinToolManageService.set_default_provider(
-            tenant_id=current_tenant_id,
-            user_id=current_user.id,
-            provider=provider,
-            id=payload.id,
-            account=current_user,
+            tenant_id=current_tenant_id, user_id=current_user.id, provider=provider, id=payload.id
         )
 
 

api/controllers/service_api/app/annotation.py

@@ -1,16 +1,16 @@
 from typing import Literal
 
 from flask import request
-from flask_restx import Resource
+from flask_restx import Namespace, Resource, fields
 from flask_restx.api import HTTPStatus
-from pydantic import BaseModel, Field, TypeAdapter
+from pydantic import BaseModel, Field
 
 from controllers.common.schema import register_schema_models
 from controllers.console.wraps import edit_permission_required
 from controllers.service_api import service_api_ns
 from controllers.service_api.wraps import validate_app_token
 from extensions.ext_redis import redis_client
-from fields.annotation_fields import Annotation, AnnotationList
+from fields.annotation_fields import annotation_fields, build_annotation_model
 from models.model import App
 from services.annotation_service import AppAnnotationService
 
@@ -26,9 +26,7 @@ class AnnotationReplyActionPayload(BaseModel):
     embedding_model_name: str = Field(description="Embedding model name")
 
 
-register_schema_models(
-    service_api_ns, AnnotationCreatePayload, AnnotationReplyActionPayload, Annotation, AnnotationList
-)
+register_schema_models(service_api_ns, AnnotationCreatePayload, AnnotationReplyActionPayload)
 
 
 @service_api_ns.route("/apps/annotation-reply/<string:action>")
@@ -47,11 +45,10 @@ class AnnotationReplyActionApi(Resource):
     def post(self, app_model: App, action: Literal["enable", "disable"]):
         """Enable or disable annotation reply feature."""
         args = AnnotationReplyActionPayload.model_validate(service_api_ns.payload or {}).model_dump()
-        match action:
-            case "enable":
-                result = AppAnnotationService.enable_app_annotation(args, app_model.id)
-            case "disable":
-                result = AppAnnotationService.disable_app_annotation(app_model.id)
+        if action == "enable":
+            result = AppAnnotationService.enable_app_annotation(args, app_model.id)
+        elif action == "disable":
+            result = AppAnnotationService.disable_app_annotation(app_model.id)
         return result, 200
 
 
@@ -85,6 +82,23 @@ class AnnotationReplyActionStatusApi(Resource):
         return {"job_id": job_id, "job_status": job_status, "error_msg": error_msg}, 200
 
 
+# Define annotation list response model
+annotation_list_fields = {
+    "data": fields.List(fields.Nested(annotation_fields)),
+    "has_more": fields.Boolean,
+    "limit": fields.Integer,
+    "total": fields.Integer,
+    "page": fields.Integer,
+}
+
+
+def build_annotation_list_model(api_or_ns: Namespace):
+    """Build the annotation list model for the API or Namespace."""
+    copied_annotation_list_fields = annotation_list_fields.copy()
+    copied_annotation_list_fields["data"] = fields.List(fields.Nested(build_annotation_model(api_or_ns)))
+    return api_or_ns.model("AnnotationList", copied_annotation_list_fields)
+
+
 @service_api_ns.route("/apps/annotations")
 class AnnotationListApi(Resource):
     @service_api_ns.doc("list_annotations")
@@ -95,12 +109,8 @@ class AnnotationListApi(Resource):
             401: "Unauthorized - invalid API token",
         }
     )
-    @service_api_ns.response(
-        200,
-        "Annotations retrieved successfully",
-        service_api_ns.models[AnnotationList.__name__],
-    )
     @validate_app_token
+    @service_api_ns.marshal_with(build_annotation_list_model(service_api_ns))
     def get(self, app_model: App):
         """List annotations for the application."""
         page = request.args.get("page", default=1, type=int)
@@ -108,15 +118,13 @@ class AnnotationListApi(Resource):
         keyword = request.args.get("keyword", default="", type=str)
 
         annotation_list, total = AppAnnotationService.get_annotation_list_by_app_id(app_model.id, page, limit, keyword)
-        annotation_models = TypeAdapter(list[Annotation]).validate_python(annotation_list, from_attributes=True)
-        response = AnnotationList(
-            data=annotation_models,
-            has_more=len(annotation_list) == limit,
-            limit=limit,
-            total=total,
-            page=page,
-        )
-        return response.model_dump(mode="json")
+        return {
+            "data": annotation_list,
+            "has_more": len(annotation_list) == limit,
+            "limit": limit,
+            "total": total,
+            "page": page,
+        }
 
     @service_api_ns.expect(service_api_ns.models[AnnotationCreatePayload.__name__])
     @service_api_ns.doc("create_annotation")
@@ -127,18 +135,13 @@ class AnnotationListApi(Resource):
             401: "Unauthorized - invalid API token",
         }
     )
-    @service_api_ns.response(
-        HTTPStatus.CREATED,
-        "Annotation created successfully",
-        service_api_ns.models[Annotation.__name__],
-    )
     @validate_app_token
+    @service_api_ns.marshal_with(build_annotation_model(service_api_ns), code=HTTPStatus.CREATED)
     def post(self, app_model: App):
         """Create a new annotation."""
         args = AnnotationCreatePayload.model_validate(service_api_ns.payload or {}).model_dump()
         annotation = AppAnnotationService.insert_app_annotation_directly(args, app_model.id)
-        response = Annotation.model_validate(annotation, from_attributes=True)
-        return response.model_dump(mode="json"), HTTPStatus.CREATED
+        return annotation, 201
 
 
 @service_api_ns.route("/apps/annotations/<uuid:annotation_id>")
@@ -155,19 +158,14 @@ class AnnotationUpdateDeleteApi(Resource):
             404: "Annotation not found",
         }
     )
-    @service_api_ns.response(
-        200,
-        "Annotation updated successfully",
-        service_api_ns.models[Annotation.__name__],
-    )
     @validate_app_token
     @edit_permission_required
+    @service_api_ns.marshal_with(build_annotation_model(service_api_ns))
     def put(self, app_model: App, annotation_id: str):
         """Update an existing annotation."""
         args = AnnotationCreatePayload.model_validate(service_api_ns.payload or {}).model_dump()
         annotation = AppAnnotationService.update_app_annotation_directly(args, app_model.id, annotation_id)
-        response = Annotation.model_validate(annotation, from_attributes=True)
-        return response.model_dump(mode="json")
+        return annotation
 
     @service_api_ns.doc("delete_annotation")
     @service_api_ns.doc(description="Delete an annotation")

api/controllers/service_api/dataset/dataset.py

@@ -17,7 +17,7 @@ from controllers.service_api.wraps import (
 from core.model_runtime.entities.model_entities import ModelType
 from core.provider_manager import ProviderManager
 from fields.dataset_fields import dataset_detail_fields
-from fields.tag_fields import DataSetTag
+from fields.tag_fields import build_dataset_tag_fields
 from libs.login import current_user
 from models.account import Account
 from models.dataset import DatasetPermissionEnum
@@ -114,7 +114,6 @@ register_schema_models(
     TagBindingPayload,
     TagUnbindingPayload,
     DatasetListQuery,
-    DataSetTag,
 )
 
 
@@ -481,14 +480,15 @@ class DatasetTagsApi(DatasetApiResource):
             401: "Unauthorized - invalid API token",
         }
     )
+    @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
     def get(self, _):
         """Get all knowledge type tags."""
         assert isinstance(current_user, Account)
         cid = current_user.current_tenant_id
         assert cid is not None
         tags = TagService.get_tags("knowledge", cid)
-        tag_models = TypeAdapter(list[DataSetTag]).validate_python(tags, from_attributes=True)
-        return [tag.model_dump(mode="json") for tag in tag_models], 200
+
+        return tags, 200
 
     @service_api_ns.expect(service_api_ns.models[TagCreatePayload.__name__])
     @service_api_ns.doc("create_dataset_tag")
@@ -500,6 +500,7 @@ class DatasetTagsApi(DatasetApiResource):
             403: "Forbidden - insufficient permissions",
         }
     )
+    @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
     def post(self, _):
         """Add a knowledge type tag."""
         assert isinstance(current_user, Account)
@@ -509,9 +510,7 @@ class DatasetTagsApi(DatasetApiResource):
         payload = TagCreatePayload.model_validate(service_api_ns.payload or {})
         tag = TagService.save_tags({"name": payload.name, "type": "knowledge"})
 
-        response = DataSetTag.model_validate(
-            {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0}
-        ).model_dump(mode="json")
+        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": 0}
         return response, 200
 
     @service_api_ns.expect(service_api_ns.models[TagUpdatePayload.__name__])
@@ -524,6 +523,7 @@ class DatasetTagsApi(DatasetApiResource):
             403: "Forbidden - insufficient permissions",
         }
     )
+    @service_api_ns.marshal_with(build_dataset_tag_fields(service_api_ns))
     def patch(self, _):
         assert isinstance(current_user, Account)
         if not (current_user.has_edit_permission or current_user.is_dataset_editor):
@@ -536,9 +536,8 @@ class DatasetTagsApi(DatasetApiResource):
 
         binding_count = TagService.get_tag_binding_count(tag_id)
 
-        response = DataSetTag.model_validate(
-            {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count}
-        ).model_dump(mode="json")
+        response = {"id": tag.id, "name": tag.name, "type": tag.type, "binding_count": binding_count}
+
         return response, 200
 
     @service_api_ns.expect(service_api_ns.models[TagDeletePayload.__name__])

api/controllers/service_api/dataset/metadata.py

@@ -168,11 +168,10 @@ class DatasetMetadataBuiltInFieldActionServiceApi(DatasetApiResource):
             raise NotFound("Dataset not found.")
         DatasetService.check_dataset_permission(dataset, current_user)
 
-        match action:
-            case "enable":
-                MetadataService.enable_built_in_field(dataset)
-            case "disable":
-                MetadataService.disable_built_in_field(dataset)
+        if action == "enable":
+            MetadataService.enable_built_in_field(dataset)
+        elif action == "disable":
+            MetadataService.disable_built_in_field(dataset)
         return {"result": "success"}, 200
 
 

api/controllers/service_api/wraps.py

@@ -73,14 +73,14 @@ def validate_app_token(view: Callable[P, R] | None = None, *, fetch_user_arg: Fe
 
             # If caller needs end-user context, attach EndUser to current_user
             if fetch_user_arg:
-                user_id = None
-                match fetch_user_arg.fetch_from:
-                    case WhereisUserArg.QUERY:
-                        user_id = request.args.get("user")
-                    case WhereisUserArg.JSON:
-                        user_id = request.get_json().get("user")
-                    case WhereisUserArg.FORM:
-                        user_id = request.form.get("user")
+                if fetch_user_arg.fetch_from == WhereisUserArg.QUERY:
+                    user_id = request.args.get("user")
+                elif fetch_user_arg.fetch_from == WhereisUserArg.JSON:
+                    user_id = request.get_json().get("user")
+                elif fetch_user_arg.fetch_from == WhereisUserArg.FORM:
+                    user_id = request.form.get("user")
+                else:
+                    user_id = None
 
                 if not user_id and fetch_user_arg.required:
                     raise ValueError("Arg user must be provided.")

api/core/agent/base_agent_runner.py

@@ -79,7 +79,7 @@ class BaseAgentRunner(AppRunner):
         self.model_instance = model_instance
 
         # init callback
-        self.agent_callback = DifyAgentCallbackHandler(tenant_id=tenant_id)
+        self.agent_callback = DifyAgentCallbackHandler()
         # init dataset tools
         hit_callback = DatasetIndexToolCallbackHandler(
             queue_manager=queue_manager,

api/core/app/app_config/easy_ui_based_app/agent/manager.py

@@ -14,17 +14,16 @@ class AgentConfigManager:
             agent_dict = config.get("agent_mode", {})
             agent_strategy = agent_dict.get("strategy", "cot")
 
-            match agent_strategy:
-                case "function_call":
+            if agent_strategy == "function_call":
+                strategy = AgentEntity.Strategy.FUNCTION_CALLING
+            elif agent_strategy in {"cot", "react"}:
+                strategy = AgentEntity.Strategy.CHAIN_OF_THOUGHT
+            else:
+                # old configs, try to detect default strategy
+                if config["model"]["provider"] == "openai":
                     strategy = AgentEntity.Strategy.FUNCTION_CALLING
-                case "cot" | "react":
+                else:
                     strategy = AgentEntity.Strategy.CHAIN_OF_THOUGHT
-                case _:
-                    # old configs, try to detect default strategy
-                    if config["model"]["provider"] == "openai":
-                        strategy = AgentEntity.Strategy.FUNCTION_CALLING
-                    else:
-                        strategy = AgentEntity.Strategy.CHAIN_OF_THOUGHT
 
             agent_tools = []
             for tool in agent_dict.get("tools", []):

api/core/app/apps/advanced_chat/generate_task_pipeline.py

@@ -63,8 +63,6 @@ from core.base.tts import AppGeneratorTTSPublisher, AudioTrunk
 from core.model_runtime.entities.llm_entities import LLMUsage
 from core.model_runtime.utils.encoders import jsonable_encoder
 from core.ops.ops_trace_manager import TraceQueueManager
-from core.telemetry import TelemetryContext, TelemetryEvent, TraceTaskName
-from core.telemetry import emit as telemetry_emit
 from core.workflow.enums import WorkflowExecutionStatus
 from core.workflow.nodes import NodeType
 from core.workflow.repositories.draft_variable_repository import DraftVariableSaverFactory
@@ -566,6 +564,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         **kwargs,
     ) -> Generator[StreamResponse, None, None]:
         """Handle stop events."""
+        _ = trace_manager
         resolved_state = None
         if self._workflow_run_id:
             resolved_state = self._resolve_graph_runtime_state(graph_runtime_state)
@@ -580,7 +579,8 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
             )
 
             with self._database_session() as session:
-                self._save_message(session=session, graph_runtime_state=resolved_state, trace_manager=trace_manager)
+                # Save message
+                self._save_message(session=session, graph_runtime_state=resolved_state)
 
             yield workflow_finish_resp
         elif event.stopped_by in (
@@ -589,7 +589,8 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         ):
             # When hitting input-moderation or annotation-reply, the workflow will not start
             with self._database_session() as session:
-                self._save_message(session=session, trace_manager=trace_manager)
+                # Save message
+                self._save_message(session=session)
 
         yield self._message_end_to_stream_response()
 
@@ -598,7 +599,6 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         event: QueueAdvancedChatMessageEndEvent,
         *,
         graph_runtime_state: GraphRuntimeState | None = None,
-        trace_manager: TraceQueueManager | None = None,
         **kwargs,
     ) -> Generator[StreamResponse, None, None]:
         """Handle advanced chat message end events."""
@@ -616,7 +616,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
 
         # Save message
         with self._database_session() as session:
-            self._save_message(session=session, graph_runtime_state=resolved_state, trace_manager=trace_manager)
+            self._save_message(session=session, graph_runtime_state=resolved_state)
 
         yield self._message_end_to_stream_response()
 
@@ -770,13 +770,7 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         if self._conversation_name_generate_thread:
             logger.debug("Conversation name generation running as daemon thread")
 
-    def _save_message(
-        self,
-        *,
-        session: Session,
-        graph_runtime_state: GraphRuntimeState | None = None,
-        trace_manager: TraceQueueManager | None = None,
-    ):
+    def _save_message(self, *, session: Session, graph_runtime_state: GraphRuntimeState | None = None):
         message = self._get_message(session=session)
 
         # If there are assistant files, remove markdown image links from answer
@@ -832,22 +826,6 @@ class AdvancedChatAppGenerateTaskPipeline(GraphRuntimeStateSupport):
         ]
         session.add_all(message_files)
 
-        if trace_manager:
-            telemetry_emit(
-                TelemetryEvent(
-                    name=TraceTaskName.MESSAGE_TRACE,
-                    context=TelemetryContext(
-                        tenant_id=self._application_generate_entity.app_config.tenant_id,
-                        app_id=self._application_generate_entity.app_config.app_id,
-                    ),
-                    payload={
-                        "conversation_id": str(message.conversation_id),
-                        "message_id": str(message.id),
-                    },
-                ),
-                trace_manager=trace_manager,
-            )
-
     def _seed_graph_runtime_state_from_queue_manager(self) -> None:
         """Bootstrap the cached runtime state from the queue manager when present."""
         candidate = self._base_task_pipeline.queue_manager.graph_runtime_state

api/core/app/apps/common/workflow_response_converter.py

@@ -250,7 +250,7 @@ class WorkflowResponseConverter:
             data=WorkflowFinishStreamResponse.Data(
                 id=run_id,
                 workflow_id=workflow_id,
-                status=status,
+                status=status.value,
                 outputs=encoded_outputs,
                 error=error,
                 elapsed_time=elapsed_time,
@@ -340,13 +340,13 @@ class WorkflowResponseConverter:
         metadata = self._merge_metadata(event.execution_metadata, snapshot)
 
         if isinstance(event, QueueNodeSucceededEvent):
-            status = WorkflowNodeExecutionStatus.SUCCEEDED
+            status = WorkflowNodeExecutionStatus.SUCCEEDED.value
             error_message = event.error
         elif isinstance(event, QueueNodeFailedEvent):
-            status = WorkflowNodeExecutionStatus.FAILED
+            status = WorkflowNodeExecutionStatus.FAILED.value
             error_message = event.error
         else:
-            status = WorkflowNodeExecutionStatus.EXCEPTION
+            status = WorkflowNodeExecutionStatus.EXCEPTION.value
             error_message = event.error
 
         return NodeFinishStreamResponse(
@@ -413,7 +413,7 @@ class WorkflowResponseConverter:
                 process_data_truncated=process_data_truncated,
                 outputs=outputs,
                 outputs_truncated=outputs_truncated,
-                status=WorkflowNodeExecutionStatus.RETRY,
+                status=WorkflowNodeExecutionStatus.RETRY.value,
                 error=event.error,
                 elapsed_time=elapsed_time,
                 execution_metadata=metadata,

api/core/app/apps/pipeline/pipeline_generator.py

@@ -120,7 +120,7 @@ class PipelineGenerator(BaseAppGenerator):
                 raise ValueError("Pipeline dataset is required")
         inputs: Mapping[str, Any] = args["inputs"]
         start_node_id: str = args["start_node_id"]
-        datasource_type = DatasourceProviderType(args["datasource_type"])
+        datasource_type: str = args["datasource_type"]
         datasource_info_list: list[Mapping[str, Any]] = self._format_datasource_info_list(
             datasource_type, args["datasource_info_list"], pipeline, workflow, start_node_id, user
         )
@@ -660,7 +660,7 @@ class PipelineGenerator(BaseAppGenerator):
         tenant_id: str,
         dataset_id: str,
         built_in_field_enabled: bool,
-        datasource_type: DatasourceProviderType,
+        datasource_type: str,
         datasource_info: Mapping[str, Any],
         created_from: str,
         position: int,
@@ -668,17 +668,17 @@ class PipelineGenerator(BaseAppGenerator):
         batch: str,
         document_form: str,
     ):
-        match datasource_type:
-            case DatasourceProviderType.LOCAL_FILE:
-                name = datasource_info.get("name", "untitled")
-            case DatasourceProviderType.ONLINE_DOCUMENT:
-                name = datasource_info.get("page", {}).get("page_name", "untitled")
-            case DatasourceProviderType.WEBSITE_CRAWL:
-                name = datasource_info.get("title", "untitled")
-            case DatasourceProviderType.ONLINE_DRIVE:
-                name = datasource_info.get("name", "untitled")
-            case _:
-                raise ValueError(f"Unsupported datasource type: {datasource_type}")
+        if datasource_type == "local_file":
+            name = datasource_info.get("name", "untitled")
+        elif datasource_type == "online_document":
+            name = datasource_info.get("page", {}).get("page_name", "untitled")
+        elif datasource_type == "website_crawl":
+            name = datasource_info.get("title", "untitled")
+        elif datasource_type == "online_drive":
+            name = datasource_info.get("name", "untitled")
+        else:
+            raise ValueError(f"Unsupported datasource type: {datasource_type}")
+
         document = Document(
             tenant_id=tenant_id,
             dataset_id=dataset_id,
@@ -706,7 +706,7 @@ class PipelineGenerator(BaseAppGenerator):
 
     def _format_datasource_info_list(
         self,
-        datasource_type: DatasourceProviderType,
+        datasource_type: str,
         datasource_info_list: list[Mapping[str, Any]],
         pipeline: Pipeline,
         workflow: Workflow,
@@ -716,7 +716,7 @@ class PipelineGenerator(BaseAppGenerator):
         """
         Format datasource info list.
         """
-        if datasource_type == DatasourceProviderType.ONLINE_DRIVE:
+        if datasource_type == "online_drive":
             all_files: list[Mapping[str, Any]] = []
             datasource_node_data = None
             datasource_nodes = workflow.graph_dict.get("nodes", [])

api/core/app/apps/workflow/app_generator.py

@@ -147,12 +147,9 @@ class WorkflowAppGenerator(BaseAppGenerator):
 
         inputs: Mapping[str, Any] = args["inputs"]
 
-        extras: dict[str, Any] = {
+        extras = {
             **extract_external_trace_id_from_args(args),
         }
-        parent_trace_context = args.get("_parent_trace_context")
-        if parent_trace_context:
-            extras["parent_trace_context"] = parent_trace_context
         workflow_run_id = str(uuid.uuid4())
         # FIXME (Yeuoly): we need to remove the SKIP_PREPARE_USER_INPUTS_KEY from the args
         # trigger shouldn't prepare user inputs

api/core/app/entities/task_entities.py

@@ -7,7 +7,7 @@ from pydantic import BaseModel, ConfigDict, Field
 from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage
 from core.rag.entities.citation_metadata import RetrievalSourceMetadata
 from core.workflow.entities import AgentNodeStrategyInit
-from core.workflow.enums import WorkflowExecutionStatus, WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
+from core.workflow.enums import WorkflowNodeExecutionMetadataKey, WorkflowNodeExecutionStatus
 
 
 class AnnotationReplyAccount(BaseModel):
@@ -223,7 +223,7 @@ class WorkflowFinishStreamResponse(StreamResponse):
 
         id: str
         workflow_id: str
-        status: WorkflowExecutionStatus
+        status: str
         outputs: Mapping[str, Any] | None = None
         error: str | None = None
         elapsed_time: float
@@ -311,7 +311,7 @@ class NodeFinishStreamResponse(StreamResponse):
         process_data_truncated: bool = False
         outputs: Mapping[str, Any] | None = None
         outputs_truncated: bool = True
-        status: WorkflowNodeExecutionStatus
+        status: str
         error: str | None = None
         elapsed_time: float
         execution_metadata: Mapping[WorkflowNodeExecutionMetadataKey, Any] | None = None
@@ -375,7 +375,7 @@ class NodeRetryStreamResponse(StreamResponse):
         process_data_truncated: bool = False
         outputs: Mapping[str, Any] | None = None
         outputs_truncated: bool = False
-        status: WorkflowNodeExecutionStatus
+        status: str
         error: str | None = None
         elapsed_time: float
         execution_metadata: Mapping[WorkflowNodeExecutionMetadataKey, Any] | None = None
@@ -719,7 +719,7 @@ class WorkflowAppBlockingResponse(AppBlockingResponse):
 
         id: str
         workflow_id: str
-        status: WorkflowExecutionStatus
+        status: str
         outputs: Mapping[str, Any] | None = None
         error: str | None = None
         elapsed_time: float

api/core/app/task_pipeline/easy_ui_based_generate_task_pipeline.py

@@ -45,8 +45,6 @@ from core.app.entities.task_entities import (
 from core.app.task_pipeline.based_generate_task_pipeline import BasedGenerateTaskPipeline
 from core.app.task_pipeline.message_cycle_manager import MessageCycleManager
 from core.base.tts import AppGeneratorTTSPublisher, AudioTrunk
-from core.file import helpers as file_helpers
-from core.file.enums import FileTransferMethod
 from core.model_manager import ModelInstance
 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta, LLMUsage
 from core.model_runtime.entities.message_entities import (
@@ -54,16 +52,14 @@ from core.model_runtime.entities.message_entities import (
     TextPromptMessageContent,
 )
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.ops.ops_trace_manager import TraceQueueManager
+from core.ops.entities.trace_entity import TraceTaskName
+from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.prompt.utils.prompt_message_util import PromptMessageUtil
 from core.prompt.utils.prompt_template_parser import PromptTemplateParser
-from core.telemetry import TelemetryContext, TelemetryEvent, TraceTaskName
-from core.telemetry import emit as telemetry_emit
-from core.tools.signature import sign_tool_file
 from events.message_event import message_was_created
 from extensions.ext_database import db
 from libs.datetime_utils import naive_utc_now
-from models.model import AppMode, Conversation, Message, MessageAgentThought, MessageFile, UploadFile
+from models.model import AppMode, Conversation, Message, MessageAgentThought
 
 logger = logging.getLogger(__name__)
 
@@ -413,19 +409,10 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline):
         message.message_metadata = self._task_state.metadata.model_dump_json()
 
         if trace_manager:
-            telemetry_emit(
-                TelemetryEvent(
-                    name=TraceTaskName.MESSAGE_TRACE,
-                    context=TelemetryContext(
-                        tenant_id=self._application_generate_entity.app_config.tenant_id,
-                        app_id=self._application_generate_entity.app_config.app_id,
-                    ),
-                    payload={
-                        "conversation_id": self._conversation_id,
-                        "message_id": self._message_id,
-                    },
-                ),
-                trace_manager=trace_manager,
+            trace_manager.add_trace_task(
+                TraceTask(
+                    TraceTaskName.MESSAGE_TRACE, conversation_id=self._conversation_id, message_id=self._message_id
+                )
             )
 
         message_was_created.send(
@@ -476,85 +463,6 @@ class EasyUIBasedGenerateTaskPipeline(BasedGenerateTaskPipeline):
             metadata=metadata_dict,
         )
 
-    def _record_files(self):
-        with Session(db.engine, expire_on_commit=False) as session:
-            message_files = session.scalars(select(MessageFile).where(MessageFile.message_id == self._message_id)).all()
-            if not message_files:
-                return None
-
-            files_list = []
-            upload_file_ids = [
-                mf.upload_file_id
-                for mf in message_files
-                if mf.transfer_method == FileTransferMethod.LOCAL_FILE and mf.upload_file_id
-            ]
-            upload_files_map = {}
-            if upload_file_ids:
-                upload_files = session.scalars(select(UploadFile).where(UploadFile.id.in_(upload_file_ids))).all()
-                upload_files_map = {uf.id: uf for uf in upload_files}
-
-            for message_file in message_files:
-                upload_file = None
-                if message_file.transfer_method == FileTransferMethod.LOCAL_FILE and message_file.upload_file_id:
-                    upload_file = upload_files_map.get(message_file.upload_file_id)
-
-                url = None
-                filename = "file"
-                mime_type = "application/octet-stream"
-                size = 0
-                extension = ""
-
-                if message_file.transfer_method == FileTransferMethod.REMOTE_URL:
-                    url = message_file.url
-                    if message_file.url:
-                        filename = message_file.url.split("/")[-1].split("?")[0]  # Remove query params
-                elif message_file.transfer_method == FileTransferMethod.LOCAL_FILE:
-                    if upload_file:
-                        url = file_helpers.get_signed_file_url(upload_file_id=str(upload_file.id))
-                        filename = upload_file.name
-                        mime_type = upload_file.mime_type or "application/octet-stream"
-                        size = upload_file.size or 0
-                        extension = f".{upload_file.extension}" if upload_file.extension else ""
-                    elif message_file.upload_file_id:
-                        # Fallback: generate URL even if upload_file not found
-                        url = file_helpers.get_signed_file_url(upload_file_id=str(message_file.upload_file_id))
-                elif message_file.transfer_method == FileTransferMethod.TOOL_FILE and message_file.url:
-                    # For tool files, use URL directly if it's HTTP, otherwise sign it
-                    if message_file.url.startswith("http"):
-                        url = message_file.url
-                        filename = message_file.url.split("/")[-1].split("?")[0]
-                    else:
-                        # Extract tool file id and extension from URL
-                        url_parts = message_file.url.split("/")
-                        if url_parts:
-                            file_part = url_parts[-1].split("?")[0]  # Remove query params first
-                            # Use rsplit to correctly handle filenames with multiple dots
-                            if "." in file_part:
-                                tool_file_id, ext = file_part.rsplit(".", 1)
-                                extension = f".{ext}"
-                            else:
-                                tool_file_id = file_part
-                                extension = ".bin"
-                            url = sign_tool_file(tool_file_id=tool_file_id, extension=extension)
-                            filename = file_part
-
-                transfer_method_value = message_file.transfer_method
-                remote_url = message_file.url if message_file.transfer_method == FileTransferMethod.REMOTE_URL else ""
-                file_dict = {
-                    "related_id": message_file.id,
-                    "extension": extension,
-                    "filename": filename,
-                    "size": size,
-                    "mime_type": mime_type,
-                    "transfer_method": transfer_method_value,
-                    "type": message_file.type,
-                    "url": url or "",
-                    "upload_file_id": message_file.upload_file_id or message_file.id,
-                    "remote_url": remote_url,
-                }
-                files_list.append(file_dict)
-            return files_list or None
-
     def _agent_message_to_stream_response(self, answer: str, message_id: str) -> AgentMessageStreamResponse:
         """
         Agent message to stream response.

api/core/app/task_pipeline/message_cycle_manager.py

@@ -64,13 +64,7 @@ class MessageCycleManager:
 
         # Use SQLAlchemy 2.x style session.scalar(select(...))
         with session_factory.create_session() as session:
-            message_file = session.scalar(
-                select(MessageFile)
-                .where(
-                    MessageFile.message_id == message_id,
-                )
-                .where(MessageFile.belongs_to == "assistant")
-            )
+            message_file = session.scalar(select(MessageFile).where(MessageFile.message_id == message_id))
 
         if message_file:
             self._message_has_file.add(message_id)

api/core/app/workflow/layers/persistence.py

@@ -15,7 +15,8 @@ from datetime import datetime
 from typing import Any, Union
 
 from core.app.entities.app_invoke_entities import AdvancedChatAppGenerateEntity, WorkflowAppGenerateEntity
-from core.ops.ops_trace_manager import TraceQueueManager
+from core.ops.entities.trace_entity import TraceTaskName
+from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.workflow.constants import SYSTEM_VARIABLE_NODE_ID
 from core.workflow.entities import WorkflowExecution, WorkflowNodeExecution
 from core.workflow.enums import (
@@ -372,7 +373,6 @@ class WorkflowPersistenceLayer(GraphEngineLayer):
 
         self._workflow_node_execution_repository.save(domain_execution)
         self._workflow_node_execution_repository.save_execution_data(domain_execution)
-        self._enqueue_node_trace_task(domain_execution)
 
     def _fail_running_node_executions(self, *, error_message: str) -> None:
         now = naive_utc_now()
@@ -390,138 +390,17 @@ class WorkflowPersistenceLayer(GraphEngineLayer):
 
         conversation_id = self._system_variables().get(SystemVariableKey.CONVERSATION_ID.value)
         external_trace_id = None
-        parent_trace_context = None
         if isinstance(self._application_generate_entity, (WorkflowAppGenerateEntity, AdvancedChatAppGenerateEntity)):
             external_trace_id = self._application_generate_entity.extras.get("external_trace_id")
-            parent_trace_context = self._application_generate_entity.extras.get("parent_trace_context")
 
-        from core.telemetry import TelemetryContext, TelemetryEvent, TraceTaskName
-        from core.telemetry import emit as telemetry_emit
-
-        telemetry_emit(
-            TelemetryEvent(
-                name=TraceTaskName.WORKFLOW_TRACE,
-                context=TelemetryContext(
-                    tenant_id=self._application_generate_entity.app_config.tenant_id,
-                    user_id=self._trace_manager.user_id,
-                    app_id=self._application_generate_entity.app_config.app_id,
-                ),
-                payload={
-                    "workflow_execution": execution,
-                    "conversation_id": conversation_id,
-                    "user_id": self._trace_manager.user_id,
-                    "external_trace_id": external_trace_id,
-                    "parent_trace_context": parent_trace_context,
-                },
-            ),
-            trace_manager=self._trace_manager,
-        )
-
-    def _enqueue_node_trace_task(self, domain_execution: WorkflowNodeExecution) -> None:
-        if not self._trace_manager:
-            return
-
-        execution = self._get_workflow_execution()
-        meta = domain_execution.metadata or {}
-
-        parent_trace_context = None
-        if isinstance(self._application_generate_entity, (WorkflowAppGenerateEntity, AdvancedChatAppGenerateEntity)):
-            parent_trace_context = self._application_generate_entity.extras.get("parent_trace_context")
-
-        node_data: dict[str, Any] = {
-            "workflow_id": domain_execution.workflow_id,
-            "workflow_execution_id": execution.id_,
-            "tenant_id": self._application_generate_entity.app_config.tenant_id,
-            "app_id": self._application_generate_entity.app_config.app_id,
-            "node_execution_id": domain_execution.id,
-            "node_id": domain_execution.node_id,
-            "node_type": str(domain_execution.node_type.value),
-            "title": domain_execution.title,
-            "status": str(domain_execution.status.value),
-            "error": domain_execution.error,
-            "elapsed_time": domain_execution.elapsed_time,
-            "index": domain_execution.index,
-            "predecessor_node_id": domain_execution.predecessor_node_id,
-            "created_at": domain_execution.created_at,
-            "finished_at": domain_execution.finished_at,
-            "total_tokens": meta.get(WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS, 0),
-            "prompt_tokens": meta.get(WorkflowNodeExecutionMetadataKey.PROMPT_TOKENS),
-            "completion_tokens": meta.get(WorkflowNodeExecutionMetadataKey.COMPLETION_TOKENS),
-            "total_price": meta.get(WorkflowNodeExecutionMetadataKey.TOTAL_PRICE, 0.0),
-            "currency": meta.get(WorkflowNodeExecutionMetadataKey.CURRENCY),
-            "tool_name": (meta.get(WorkflowNodeExecutionMetadataKey.TOOL_INFO) or {}).get("tool_name")
-            if isinstance(meta.get(WorkflowNodeExecutionMetadataKey.TOOL_INFO), dict)
-            else None,
-            "iteration_id": meta.get(WorkflowNodeExecutionMetadataKey.ITERATION_ID),
-            "iteration_index": meta.get(WorkflowNodeExecutionMetadataKey.ITERATION_INDEX),
-            "loop_id": meta.get(WorkflowNodeExecutionMetadataKey.LOOP_ID),
-            "loop_index": meta.get(WorkflowNodeExecutionMetadataKey.LOOP_INDEX),
-            "parallel_id": meta.get(WorkflowNodeExecutionMetadataKey.PARALLEL_ID),
-            "node_inputs": dict(domain_execution.inputs) if domain_execution.inputs else None,
-            "node_outputs": dict(domain_execution.outputs) if domain_execution.outputs else None,
-            "process_data": dict(domain_execution.process_data) if domain_execution.process_data else None,
-        }
-        node_data["invoke_from"] = self._application_generate_entity.invoke_from.value
-        node_data["user_id"] = self._system_variables().get(SystemVariableKey.USER_ID.value)
-
-        # Extract model info from process_data — LLM nodes store provider/model there,
-        if domain_execution.process_data:
-            if mp := domain_execution.process_data.get("model_provider"):
-                node_data["model_provider"] = mp
-            if mn := domain_execution.process_data.get("model_name"):
-                node_data["model_name"] = mn
-
-        if domain_execution.node_type.value == "knowledge-retrieval" and domain_execution.outputs:
-            results = domain_execution.outputs.get("result") or []
-            dataset_ids: list[str] = []
-            dataset_names: list[str] = []
-            for doc in results:
-                if not isinstance(doc, dict):
-                    continue
-                doc_meta = doc.get("metadata") or {}
-                did = doc_meta.get("dataset_id")
-                dname = doc_meta.get("dataset_name")
-                if did and did not in dataset_ids:
-                    dataset_ids.append(did)
-                if dname and dname not in dataset_names:
-                    dataset_names.append(dname)
-            if dataset_ids:
-                node_data["dataset_ids"] = dataset_ids
-            if dataset_names:
-                node_data["dataset_names"] = dataset_names
-
-        tool_info = meta.get(WorkflowNodeExecutionMetadataKey.TOOL_INFO)
-        if isinstance(tool_info, dict):
-            plugin_id = tool_info.get("plugin_unique_identifier")
-            if plugin_id:
-                node_data["plugin_name"] = plugin_id
-            credential_id = tool_info.get("credential_id")
-            if credential_id:
-                node_data["credential_id"] = credential_id
-                node_data["credential_provider_type"] = tool_info.get("provider_type")
-
-        conversation_id = self._system_variables().get(SystemVariableKey.CONVERSATION_ID.value)
-        if conversation_id:
-            node_data["conversation_id"] = conversation_id
-
-        if parent_trace_context:
-            node_data["parent_trace_context"] = parent_trace_context
-
-        from core.telemetry import TelemetryContext, TelemetryEvent, TraceTaskName
-        from core.telemetry import emit as telemetry_emit
-
-        telemetry_emit(
-            TelemetryEvent(
-                name=TraceTaskName.NODE_EXECUTION_TRACE,
-                context=TelemetryContext(
-                    tenant_id=node_data.get("tenant_id"),
-                    user_id=node_data.get("user_id"),
-                    app_id=node_data.get("app_id"),
-                ),
-                payload={"node_execution_data": node_data},
-            ),
-            trace_manager=self._trace_manager,
+        trace_task = TraceTask(
+            TraceTaskName.WORKFLOW_TRACE,
+            workflow_execution=execution,
+            conversation_id=conversation_id,
+            user_id=self._trace_manager.user_id,
+            external_trace_id=external_trace_id,
         )
+        self._trace_manager.add_trace_task(trace_task)
 
     def _system_variables(self) -> Mapping[str, Any]:
         runtime_state = self.graph_runtime_state

api/core/callback_handler/agent_tool_callback_handler.py

@@ -4,9 +4,8 @@ from typing import Any, TextIO, Union
 from pydantic import BaseModel
 
 from configs import dify_config
-from core.ops.ops_trace_manager import TraceQueueManager
-from core.telemetry import TelemetryContext, TelemetryEvent, TraceTaskName
-from core.telemetry import emit as telemetry_emit
+from core.ops.entities.trace_entity import TraceTaskName
+from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.tools.entities.tool_entities import ToolInvokeMessage
 
 _TEXT_COLOR_MAPPING = {
@@ -37,15 +36,13 @@ class DifyAgentCallbackHandler(BaseModel):
 
     color: str | None = ""
     current_loop: int = 1
-    tenant_id: str | None = None
 
-    def __init__(self, color: str | None = None, tenant_id: str | None = None):
+    def __init__(self, color: str | None = None):
         super().__init__()
         """Initialize callback handler."""
         # use a specific color is not specified
         self.color = color or "green"
         self.current_loop = 1
-        self.tenant_id = tenant_id
 
     def on_tool_start(
         self,
@@ -74,23 +71,15 @@ class DifyAgentCallbackHandler(BaseModel):
             print_text("\n")
 
         if trace_manager:
-            telemetry_emit(
-                TelemetryEvent(
-                    name=TraceTaskName.TOOL_TRACE,
-                    context=TelemetryContext(
-                        tenant_id=self.tenant_id,
-                        app_id=trace_manager.app_id,
-                        user_id=trace_manager.user_id,
-                    ),
-                    payload={
-                        "message_id": message_id,
-                        "tool_name": tool_name,
-                        "tool_inputs": tool_inputs,
-                        "tool_outputs": tool_outputs,
-                        "timer": timer,
-                    },
-                ),
-                trace_manager=trace_manager,
+            trace_manager.add_trace_task(
+                TraceTask(
+                    TraceTaskName.TOOL_TRACE,
+                    message_id=message_id,
+                    tool_name=tool_name,
+                    tool_inputs=tool_inputs,
+                    tool_outputs=tool_outputs,
+                    timer=timer,
+                )
             )
 
     def on_tool_error(self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any):

api/core/indexing_runner.py

@@ -369,78 +369,77 @@ class IndexingRunner:
         # Generate summary preview
         summary_index_setting = tmp_processing_rule.get("summary_index_setting")
         if summary_index_setting and summary_index_setting.get("enable") and preview_texts:
-            preview_texts = index_processor.generate_summary_preview(
-                tenant_id, preview_texts, summary_index_setting, doc_language
-            )
+            preview_texts = index_processor.generate_summary_preview(tenant_id, preview_texts, summary_index_setting)
 
         return IndexingEstimate(total_segments=total_segments, preview=preview_texts)
 
     def _extract(
         self, index_processor: BaseIndexProcessor, dataset_document: DatasetDocument, process_rule: dict
     ) -> list[Document]:
+        # load file
+        if dataset_document.data_source_type not in {"upload_file", "notion_import", "website_crawl"}:
+            return []
+
         data_source_info = dataset_document.data_source_info_dict
         text_docs = []
-        match dataset_document.data_source_type:
-            case "upload_file":
-                if not data_source_info or "upload_file_id" not in data_source_info:
-                    raise ValueError("no upload file found")
-                stmt = select(UploadFile).where(UploadFile.id == data_source_info["upload_file_id"])
-                file_detail = db.session.scalars(stmt).one_or_none()
+        if dataset_document.data_source_type == "upload_file":
+            if not data_source_info or "upload_file_id" not in data_source_info:
+                raise ValueError("no upload file found")
+            stmt = select(UploadFile).where(UploadFile.id == data_source_info["upload_file_id"])
+            file_detail = db.session.scalars(stmt).one_or_none()
 
-                if file_detail:
-                    extract_setting = ExtractSetting(
-                        datasource_type=DatasourceType.FILE,
-                        upload_file=file_detail,
-                        document_model=dataset_document.doc_form,
-                    )
-                    text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
-            case "notion_import":
-                if (
-                    not data_source_info
-                    or "notion_workspace_id" not in data_source_info
-                    or "notion_page_id" not in data_source_info
-                ):
-                    raise ValueError("no notion import info found")
+            if file_detail:
                 extract_setting = ExtractSetting(
-                    datasource_type=DatasourceType.NOTION,
-                    notion_info=NotionInfo.model_validate(
-                        {
-                            "credential_id": data_source_info.get("credential_id"),
-                            "notion_workspace_id": data_source_info["notion_workspace_id"],
-                            "notion_obj_id": data_source_info["notion_page_id"],
-                            "notion_page_type": data_source_info["type"],
-                            "document": dataset_document,
-                            "tenant_id": dataset_document.tenant_id,
-                        }
-                    ),
+                    datasource_type=DatasourceType.FILE,
+                    upload_file=file_detail,
                     document_model=dataset_document.doc_form,
                 )
                 text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
-            case "website_crawl":
-                if (
-                    not data_source_info
-                    or "provider" not in data_source_info
-                    or "url" not in data_source_info
-                    or "job_id" not in data_source_info
-                ):
-                    raise ValueError("no website import info found")
-                extract_setting = ExtractSetting(
-                    datasource_type=DatasourceType.WEBSITE,
-                    website_info=WebsiteInfo.model_validate(
-                        {
-                            "provider": data_source_info["provider"],
-                            "job_id": data_source_info["job_id"],
-                            "tenant_id": dataset_document.tenant_id,
-                            "url": data_source_info["url"],
-                            "mode": data_source_info["mode"],
-                            "only_main_content": data_source_info["only_main_content"],
-                        }
-                    ),
-                    document_model=dataset_document.doc_form,
-                )
-                text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
-            case _:
-                return []
+        elif dataset_document.data_source_type == "notion_import":
+            if (
+                not data_source_info
+                or "notion_workspace_id" not in data_source_info
+                or "notion_page_id" not in data_source_info
+            ):
+                raise ValueError("no notion import info found")
+            extract_setting = ExtractSetting(
+                datasource_type=DatasourceType.NOTION,
+                notion_info=NotionInfo.model_validate(
+                    {
+                        "credential_id": data_source_info.get("credential_id"),
+                        "notion_workspace_id": data_source_info["notion_workspace_id"],
+                        "notion_obj_id": data_source_info["notion_page_id"],
+                        "notion_page_type": data_source_info["type"],
+                        "document": dataset_document,
+                        "tenant_id": dataset_document.tenant_id,
+                    }
+                ),
+                document_model=dataset_document.doc_form,
+            )
+            text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
+        elif dataset_document.data_source_type == "website_crawl":
+            if (
+                not data_source_info
+                or "provider" not in data_source_info
+                or "url" not in data_source_info
+                or "job_id" not in data_source_info
+            ):
+                raise ValueError("no website import info found")
+            extract_setting = ExtractSetting(
+                datasource_type=DatasourceType.WEBSITE,
+                website_info=WebsiteInfo.model_validate(
+                    {
+                        "provider": data_source_info["provider"],
+                        "job_id": data_source_info["job_id"],
+                        "tenant_id": dataset_document.tenant_id,
+                        "url": data_source_info["url"],
+                        "mode": data_source_info["mode"],
+                        "only_main_content": data_source_info["only_main_content"],
+                    }
+                ),
+                document_model=dataset_document.doc_form,
+            )
+            text_docs = index_processor.extract(extract_setting, process_rule_mode=process_rule["mode"])
         # update document status to splitting
         self._update_document_index_status(
             document_id=dataset_document.id,

api/core/llm_generator/entities.py

@@ -9,7 +9,6 @@ class RuleGeneratePayload(BaseModel):
     instruction: str = Field(..., description="Rule generation instruction")
     model_config_data: ModelConfig = Field(..., alias="model_config", description="Model configuration")
     no_variable: bool = Field(default=False, description="Whether to exclude variables")
-    app_id: str | None = Field(default=None, description="App ID for prompt generation tracing")
 
 
 class RuleCodeGeneratePayload(RuleGeneratePayload):
@@ -19,4 +18,3 @@ class RuleCodeGeneratePayload(RuleGeneratePayload):
 class RuleStructuredOutputPayload(BaseModel):
     instruction: str = Field(..., description="Structured output generation instruction")
     model_config_data: ModelConfig = Field(..., alias="model_config", description="Model configuration")
-    app_id: str | None = Field(default=None, description="App ID for prompt generation tracing")

api/core/llm_generator/llm_generator.py

@@ -7,6 +7,7 @@ from typing import Protocol, cast
 import json_repair
 
 from core.app.app_config.entities import ModelConfig
+from core.llm_generator.entities import RuleCodeGeneratePayload, RuleGeneratePayload, RuleStructuredOutputPayload
 from core.llm_generator.output_parser.rule_config_generator import RuleConfigGeneratorOutputParser
 from core.llm_generator.output_parser.suggested_questions_after_answer import SuggestedQuestionsAfterAnswerOutputParser
 from core.llm_generator.prompts import (
@@ -26,11 +27,10 @@ from core.model_runtime.entities.llm_entities import LLMResult
 from core.model_runtime.entities.message_entities import PromptMessage, SystemPromptMessage, UserPromptMessage
 from core.model_runtime.entities.model_entities import ModelType
 from core.model_runtime.errors.invoke import InvokeAuthorizationError, InvokeError
-from core.ops.entities.trace_entity import OperationType
+from core.ops.entities.trace_entity import TraceTaskName
+from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.ops.utils import measure_time
 from core.prompt.utils.prompt_template_parser import PromptTemplateParser
-from core.telemetry import TelemetryContext, TelemetryEvent, TraceTaskName
-from core.telemetry import emit as telemetry_emit
 from core.workflow.entities.workflow_node_execution import WorkflowNodeExecutionMetadataKey
 from extensions.ext_database import db
 from extensions.ext_storage import storage
@@ -74,7 +74,7 @@ class LLMGenerator:
                 prompt_messages=list(prompts), model_parameters={"max_tokens": 500, "temperature": 1}, stream=False
             )
         answer = response.message.get_text_content()
-        if answer is None:
+        if answer == "":
             return ""
         try:
             result_dict = json.loads(answer)
@@ -96,17 +96,15 @@ class LLMGenerator:
             name = name[:75] + "..."
 
         # get tracing instance
-        telemetry_emit(
-            TelemetryEvent(
-                name=TraceTaskName.GENERATE_NAME_TRACE,
-                context=TelemetryContext(tenant_id=tenant_id, app_id=app_id),
-                payload={
-                    "conversation_id": conversation_id,
-                    "generate_conversation_name": name,
-                    "inputs": prompt,
-                    "timer": timer,
-                    "tenant_id": tenant_id,
-                },
+        trace_manager = TraceQueueManager(app_id=app_id)
+        trace_manager.add_trace_task(
+            TraceTask(
+                TraceTaskName.GENERATE_NAME_TRACE,
+                conversation_id=conversation_id,
+                generate_conversation_name=name,
+                inputs=prompt,
+                timer=timer,
+                tenant_id=tenant_id,
             )
         )
 
@@ -155,27 +153,19 @@ class LLMGenerator:
         return questions
 
     @classmethod
-    def generate_rule_config(
-        cls,
-        tenant_id: str,
-        instruction: str,
-        model_config: ModelConfig,
-        no_variable: bool,
-        user_id: str | None = None,
-        app_id: str | None = None,
-    ):
+    def generate_rule_config(cls, tenant_id: str, args: RuleGeneratePayload):
         output_parser = RuleConfigGeneratorOutputParser()
 
         error = ""
         error_step = ""
         rule_config = {"prompt": "", "variables": [], "opening_statement": "", "error": ""}
-        model_parameters = model_config.completion_params
-        if no_variable:
+        model_parameters = args.model_config_data.completion_params
+        if args.no_variable:
             prompt_template = PromptTemplateParser(WORKFLOW_RULE_CONFIG_PROMPT_GENERATE_TEMPLATE)
 
             prompt_generate = prompt_template.format(
                 inputs={
-                    "TASK_DESCRIPTION": instruction,
+                    "TASK_DESCRIPTION": args.instruction,
                 },
                 remove_template_variables=False,
             )
@@ -187,45 +177,26 @@ class LLMGenerator:
             model_instance = model_manager.get_model_instance(
                 tenant_id=tenant_id,
                 model_type=ModelType.LLM,
-                provider=model_config.provider,
-                model=model_config.name,
+                provider=args.model_config_data.provider,
+                model=args.model_config_data.name,
             )
 
-            llm_result = None
-            with measure_time() as timer:
-                try:
-                    llm_result = model_instance.invoke_llm(
-                        prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
-                    )
+            try:
+                response: LLMResult = model_instance.invoke_llm(
+                    prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
+                )
 
-                    rule_config["prompt"] = llm_result.message.get_text_content() or ""
+                rule_config["prompt"] = response.message.get_text_content()
 
-                except InvokeError as e:
-                    error = str(e)
-                    error_step = "generate rule config"
-                except Exception as e:
-                    logger.exception("Failed to generate rule config, model: %s", model_config.name)
-                    rule_config["error"] = str(e)
-                    error = str(e)
+            except InvokeError as e:
+                error = str(e)
+                error_step = "generate rule config"
+            except Exception as e:
+                logger.exception("Failed to generate rule config, model: %s", args.model_config_data.name)
+                rule_config["error"] = str(e)
 
             rule_config["error"] = f"Failed to {error_step}. Error: {error}" if error else ""
 
-            if user_id:
-                prompt_value = rule_config.get("prompt", "")
-                generated_output = str(prompt_value) if prompt_value else ""
-                cls._emit_prompt_generation_trace(
-                    tenant_id=tenant_id,
-                    user_id=user_id,
-                    app_id=app_id,
-                    operation_type=OperationType.RULE_GENERATE,
-                    instruction=instruction,
-                    generated_output=generated_output,
-                    llm_result=llm_result,
-                    model_config=model_config,
-                    timer=timer,
-                    error=error or None,
-                )
-
             return rule_config
 
         # get rule config prompt, parameter and statement
@@ -240,7 +211,7 @@ class LLMGenerator:
         # format the prompt_generate_prompt
         prompt_generate_prompt = prompt_template.format(
             inputs={
-                "TASK_DESCRIPTION": instruction,
+                "TASK_DESCRIPTION": args.instruction,
             },
             remove_template_variables=False,
         )
@@ -251,125 +222,84 @@ class LLMGenerator:
         model_instance = model_manager.get_model_instance(
             tenant_id=tenant_id,
             model_type=ModelType.LLM,
-            provider=model_config.provider,
-            model=model_config.name,
+            provider=args.model_config_data.provider,
+            model=args.model_config_data.name,
         )
 
-        llm_result = None
-        with measure_time() as timer:
+        try:
             try:
-                try:
-                    # the first step to generate the task prompt
-                    prompt_content: LLMResult = model_instance.invoke_llm(
-                        prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
-                    )
-                    llm_result = prompt_content
-                except InvokeError as e:
-                    error = str(e)
-                    error_step = "generate prefix prompt"
-                    rule_config["error"] = f"Failed to {error_step}. Error: {error}" if error else ""
-
-                    if user_id:
-                        cls._emit_prompt_generation_trace(
-                            tenant_id=tenant_id,
-                            user_id=user_id,
-                            app_id=app_id,
-                            operation_type=OperationType.RULE_GENERATE,
-                            instruction=instruction,
-                            generated_output="",
-                            llm_result=llm_result,
-                            model_config=model_config,
-                            timer=timer,
-                            error=error,
-                        )
-
-                    return rule_config
-
-                rule_config["prompt"] = prompt_content.message.get_text_content() or ""
-
-                if not isinstance(prompt_content.message.content, str):
-                    raise NotImplementedError("prompt content is not a string")
-                parameter_generate_prompt = parameter_template.format(
-                    inputs={
-                        "INPUT_TEXT": prompt_content.message.content,
-                    },
-                    remove_template_variables=False,
+                # the first step to generate the task prompt
+                prompt_content: LLMResult = model_instance.invoke_llm(
+                    prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
                 )
-                parameter_messages = [UserPromptMessage(content=parameter_generate_prompt)]
-
-                # the second step to generate the task_parameter and task_statement
-                statement_generate_prompt = statement_template.format(
-                    inputs={
-                        "TASK_DESCRIPTION": instruction,
-                        "INPUT_TEXT": prompt_content.message.content,
-                    },
-                    remove_template_variables=False,
-                )
-                statement_messages = [UserPromptMessage(content=statement_generate_prompt)]
-
-                try:
-                    parameter_content: LLMResult = model_instance.invoke_llm(
-                        prompt_messages=list(parameter_messages), model_parameters=model_parameters, stream=False
-                    )
-                    rule_config["variables"] = re.findall(
-                        r'"\s*([^"]+)\s*"', prompt_content.message.get_text_content() or ""
-                    )
-                except InvokeError as e:
-                    error = str(e)
-                    error_step = "generate variables"
-
-                try:
-                    statement_content: LLMResult = model_instance.invoke_llm(
-                        prompt_messages=list(statement_messages), model_parameters=model_parameters, stream=False
-                    )
-                    rule_config["opening_statement"] = statement_content.message.get_text_content() or ""
-                except InvokeError as e:
-                    error = str(e)
-                    error_step = "generate conversation opener"
-
-            except Exception as e:
-                logger.exception("Failed to generate rule config, model: %s", model_config.name)
-                rule_config["error"] = str(e)
+            except InvokeError as e:
                 error = str(e)
+                error_step = "generate prefix prompt"
+                rule_config["error"] = f"Failed to {error_step}. Error: {error}" if error else ""
+
+                return rule_config
+
+            rule_config["prompt"] = prompt_content.message.get_text_content()
+
+            parameter_generate_prompt = parameter_template.format(
+                inputs={
+                    "INPUT_TEXT": prompt_content.message.get_text_content(),
+                },
+                remove_template_variables=False,
+            )
+            parameter_messages = [UserPromptMessage(content=parameter_generate_prompt)]
+
+            # the second step to generate the task_parameter and task_statement
+            statement_generate_prompt = statement_template.format(
+                inputs={
+                    "TASK_DESCRIPTION": args.instruction,
+                    "INPUT_TEXT": prompt_content.message.get_text_content(),
+                },
+                remove_template_variables=False,
+            )
+            statement_messages = [UserPromptMessage(content=statement_generate_prompt)]
+
+            try:
+                parameter_content: LLMResult = model_instance.invoke_llm(
+                    prompt_messages=list(parameter_messages), model_parameters=model_parameters, stream=False
+                )
+                rule_config["variables"] = re.findall(r'"\s*([^"]+)\s*"', parameter_content.message.get_text_content())
+            except InvokeError as e:
+                error = str(e)
+                error_step = "generate variables"
+
+            try:
+                statement_content: LLMResult = model_instance.invoke_llm(
+                    prompt_messages=list(statement_messages), model_parameters=model_parameters, stream=False
+                )
+                rule_config["opening_statement"] = statement_content.message.get_text_content()
+            except InvokeError as e:
+                error = str(e)
+                error_step = "generate conversation opener"
+
+        except Exception as e:
+            logger.exception("Failed to generate rule config, model: %s", args.model_config_data.name)
+            rule_config["error"] = str(e)
 
         rule_config["error"] = f"Failed to {error_step}. Error: {error}" if error else ""
 
-        if user_id:
-            generated_output = rule_config.get("prompt", "")
-            cls._emit_prompt_generation_trace(
-                tenant_id=tenant_id,
-                user_id=user_id,
-                app_id=app_id,
-                operation_type=OperationType.RULE_GENERATE,
-                instruction=instruction,
-                generated_output=str(generated_output) if generated_output else "",
-                llm_result=llm_result,
-                model_config=model_config,
-                timer=timer,
-                error=error or None,
-            )
-
         return rule_config
 
     @classmethod
     def generate_code(
         cls,
         tenant_id: str,
-        instruction: str,
-        model_config: ModelConfig,
-        code_language: str = "javascript",
-        user_id: str | None = None,
-        app_id: str | None = None,
+        args: RuleCodeGeneratePayload,
     ):
-        if code_language == "python":
+        if args.code_language == "python":
             prompt_template = PromptTemplateParser(PYTHON_CODE_GENERATOR_PROMPT_TEMPLATE)
         else:
             prompt_template = PromptTemplateParser(JAVASCRIPT_CODE_GENERATOR_PROMPT_TEMPLATE)
 
         prompt = prompt_template.format(
             inputs={
-                "INSTRUCTION": instruction,
-                "CODE_LANGUAGE": code_language,
+                "INSTRUCTION": args.instruction,
+                "CODE_LANGUAGE": args.code_language,
             },
             remove_template_variables=False,
         )
@@ -378,49 +308,28 @@ class LLMGenerator:
         model_instance = model_manager.get_model_instance(
             tenant_id=tenant_id,
             model_type=ModelType.LLM,
-            provider=model_config.provider,
-            model=model_config.name,
+            provider=args.model_config_data.provider,
+            model=args.model_config_data.name,
         )
 
         prompt_messages = [UserPromptMessage(content=prompt)]
-        model_parameters = model_config.completion_params
-
-        llm_result = None
-        error = None
-        with measure_time() as timer:
-            try:
-                llm_result = model_instance.invoke_llm(
-                    prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
-                )
-
-                generated_code = llm_result.message.get_text_content() or ""
-                result = {"code": generated_code, "language": code_language, "error": ""}
-
-            except InvokeError as e:
-                error = str(e)
-                result = {"code": "", "language": code_language, "error": f"Failed to generate code. Error: {error}"}
-            except Exception as e:
-                logger.exception(
-                    "Failed to invoke LLM model, model: %s, language: %s", model_config.name, code_language
-                )
-                error = str(e)
-                result = {"code": "", "language": code_language, "error": f"An unexpected error occurred: {str(e)}"}
-
-        if user_id:
-            cls._emit_prompt_generation_trace(
-                tenant_id=tenant_id,
-                user_id=user_id,
-                app_id=app_id,
-                operation_type=OperationType.CODE_GENERATE,
-                instruction=instruction,
-                generated_output=result.get("code", ""),
-                llm_result=llm_result,
-                model_config=model_config,
-                timer=timer,
-                error=error,
+        model_parameters = args.model_config_data.completion_params
+        try:
+            response: LLMResult = model_instance.invoke_llm(
+                prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
             )
 
-        return result
+            generated_code = response.message.get_text_content()
+            return {"code": generated_code, "language": args.code_language, "error": ""}
+
+        except InvokeError as e:
+            error = str(e)
+            return {"code": "", "language": args.code_language, "error": f"Failed to generate code. Error: {error}"}
+        except Exception as e:
+            logger.exception(
+                "Failed to invoke LLM model, model: %s, language: %s", args.model_config_data.name, args.code_language
+            )
+            return {"code": "", "language": args.code_language, "error": f"An unexpected error occurred: {str(e)}"}
 
     @classmethod
     def generate_qa_document(cls, tenant_id: str, query, document_language: str):
@@ -446,81 +355,49 @@ class LLMGenerator:
             raise TypeError("Expected LLMResult when stream=False")
         response = result
 
-        answer = response.message.get_text_content() or ""
+        answer = response.message.get_text_content()
         return answer.strip()
 
     @classmethod
-    def generate_structured_output(
-        cls,
-        tenant_id: str,
-        instruction: str,
-        model_config: ModelConfig,
-        user_id: str | None = None,
-        app_id: str | None = None,
-    ):
+    def generate_structured_output(cls, tenant_id: str, args: RuleStructuredOutputPayload):
         model_manager = ModelManager()
         model_instance = model_manager.get_model_instance(
             tenant_id=tenant_id,
             model_type=ModelType.LLM,
-            provider=model_config.provider,
-            model=model_config.name,
+            provider=args.model_config_data.provider,
+            model=args.model_config_data.name,
         )
 
         prompt_messages = [
             SystemPromptMessage(content=SYSTEM_STRUCTURED_OUTPUT_GENERATE),
-            UserPromptMessage(content=instruction),
+            UserPromptMessage(content=args.instruction),
         ]
-        model_parameters = model_config.completion_params
+        model_parameters = args.model_config_data.completion_params
 
-        llm_result = None
-        error = None
-        result = {"output": "", "error": ""}
-
-        with measure_time() as timer:
-            try:
-                llm_result = model_instance.invoke_llm(
-                    prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
-                )
-
-                raw_content = llm_result.message.content
-
-                if not isinstance(raw_content, str):
-                    raise ValueError(f"LLM response content must be a string, got: {type(raw_content)}")
-
-                try:
-                    parsed_content = json.loads(raw_content)
-                except json.JSONDecodeError:
-                    parsed_content = json_repair.loads(raw_content)
-
-                if not isinstance(parsed_content, dict | list):
-                    raise ValueError(f"Failed to parse structured output from llm: {raw_content}")
-
-                generated_json_schema = json.dumps(parsed_content, indent=2, ensure_ascii=False)
-                result = {"output": generated_json_schema, "error": ""}
-
-            except InvokeError as e:
-                error = str(e)
-                result = {"output": "", "error": f"Failed to generate JSON Schema. Error: {error}"}
-            except Exception as e:
-                logger.exception("Failed to invoke LLM model, model: %s", model_config.name)
-                error = str(e)
-                result = {"output": "", "error": f"An unexpected error occurred: {str(e)}"}
-
-        if user_id:
-            cls._emit_prompt_generation_trace(
-                tenant_id=tenant_id,
-                user_id=user_id,
-                app_id=app_id,
-                operation_type=OperationType.STRUCTURED_OUTPUT,
-                instruction=instruction,
-                generated_output=result.get("output", ""),
-                llm_result=llm_result,
-                model_config=model_config,
-                timer=timer,
-                error=error,
+        try:
+            response: LLMResult = model_instance.invoke_llm(
+                prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
             )
 
-        return result
+            raw_content = response.message.get_text_content()
+
+            try:
+                parsed_content = json.loads(raw_content)
+            except json.JSONDecodeError:
+                parsed_content = json_repair.loads(raw_content)
+
+            if not isinstance(parsed_content, dict | list):
+                raise ValueError(f"Failed to parse structured output from llm: {raw_content}")
+
+            generated_json_schema = json.dumps(parsed_content, indent=2, ensure_ascii=False)
+            return {"output": generated_json_schema, "error": ""}
+
+        except InvokeError as e:
+            error = str(e)
+            return {"output": "", "error": f"Failed to generate JSON Schema. Error: {error}"}
+        except Exception as e:
+            logger.exception("Failed to invoke LLM model, model: %s", args.model_config_data.name)
+            return {"output": "", "error": f"An unexpected error occurred: {str(e)}"}
 
     @staticmethod
     def instruction_modify_legacy(
@@ -530,14 +407,12 @@ class LLMGenerator:
         instruction: str,
         model_config: ModelConfig,
         ideal_output: str | None,
-        user_id: str | None = None,
-        app_id: str | None = None,
     ):
         last_run: Message | None = (
             db.session.query(Message).where(Message.app_id == flow_id).order_by(Message.created_at.desc()).first()
         )
         if not last_run:
-            result = LLMGenerator.__instruction_modify_common(
+            return LLMGenerator.__instruction_modify_common(
                 tenant_id=tenant_id,
                 model_config=model_config,
                 last_run=None,
@@ -546,28 +421,22 @@ class LLMGenerator:
                 instruction=instruction,
                 node_type="llm",
                 ideal_output=ideal_output,
-                user_id=user_id,
-                app_id=app_id,
             )
-        else:
-            last_run_dict = {
-                "query": last_run.query,
-                "answer": last_run.answer,
-                "error": last_run.error,
-            }
-            result = LLMGenerator.__instruction_modify_common(
-                tenant_id=tenant_id,
-                model_config=model_config,
-                last_run=last_run_dict,
-                current=current,
-                error_message=str(last_run.error),
-                instruction=instruction,
-                node_type="llm",
-                ideal_output=ideal_output,
-                user_id=user_id,
-                app_id=app_id,
-            )
-        return result
+        last_run_dict = {
+            "query": last_run.query,
+            "answer": last_run.answer,
+            "error": last_run.error,
+        }
+        return LLMGenerator.__instruction_modify_common(
+            tenant_id=tenant_id,
+            model_config=model_config,
+            last_run=last_run_dict,
+            current=current,
+            error_message=str(last_run.error),
+            instruction=instruction,
+            node_type="llm",
+            ideal_output=ideal_output,
+        )
 
     @staticmethod
     def instruction_modify_workflow(
@@ -579,8 +448,6 @@ class LLMGenerator:
         model_config: ModelConfig,
         ideal_output: str | None,
         workflow_service: WorkflowServiceInterface,
-        user_id: str | None = None,
-        app_id: str | None = None,
     ):
         session = db.session()
 
@@ -611,8 +478,6 @@ class LLMGenerator:
                 instruction=instruction,
                 node_type=node_type,
                 ideal_output=ideal_output,
-                user_id=user_id,
-                app_id=app_id,
             )
 
         def agent_log_of(node_execution: WorkflowNodeExecutionModel) -> Sequence:
@@ -646,8 +511,6 @@ class LLMGenerator:
             instruction=instruction,
             node_type=last_run.node_type,
             ideal_output=ideal_output,
-            user_id=user_id,
-            app_id=app_id,
         )
 
     @staticmethod
@@ -660,8 +523,6 @@ class LLMGenerator:
         instruction: str,
         node_type: str,
         ideal_output: str | None,
-        user_id: str | None = None,
-        app_id: str | None = None,
     ):
         LAST_RUN = "{{#last_run#}}"
         CURRENT = "{{#current#}}"
@@ -701,120 +562,24 @@ class LLMGenerator:
         ]
         model_parameters = {"temperature": 0.4}
 
-        llm_result = None
-        error = None
-        result = {}
-
-        with measure_time() as timer:
-            try:
-                llm_result = model_instance.invoke_llm(
-                    prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
-                )
-
-                generated_raw = llm_result.message.get_text_content()
-                first_brace = generated_raw.find("{")
-                last_brace = generated_raw.rfind("}")
-                if first_brace == -1 or last_brace == -1 or last_brace < first_brace:
-                    raise ValueError(f"Could not find a valid JSON object in response: {generated_raw}")
-                json_str = generated_raw[first_brace : last_brace + 1]
-                data = json_repair.loads(json_str)
-                if not isinstance(data, dict):
-                    raise TypeError(f"Expected a JSON object, but got {type(data).__name__}")
-                result = data
-            except InvokeError as e:
-                error = str(e)
-                result = {"error": f"Failed to generate code. Error: {error}"}
-            except Exception as e:
-                logger.exception("Failed to invoke LLM model, model: %s", json.dumps(model_config.name), exc_info=True)
-                error = str(e)
-                result = {"error": f"An unexpected error occurred: {str(e)}"}
-
-        if user_id:
-            generated_output = ""
-            if isinstance(result, dict):
-                for key in ["prompt", "code", "output", "modified"]:
-                    if result.get(key):
-                        generated_output = str(result[key])
-                        break
-
-            LLMGenerator._emit_prompt_generation_trace(
-                tenant_id=tenant_id,
-                user_id=user_id,
-                app_id=app_id,
-                operation_type=OperationType.INSTRUCTION_MODIFY,
-                instruction=instruction,
-                generated_output=generated_output,
-                llm_result=llm_result,
-                model_config=model_config,
-                timer=timer,
-                error=error,
+        try:
+            response: LLMResult = model_instance.invoke_llm(
+                prompt_messages=list(prompt_messages), model_parameters=model_parameters, stream=False
             )
 
-        return result
-
-    @classmethod
-    def _emit_prompt_generation_trace(
-        cls,
-        tenant_id: str,
-        user_id: str,
-        app_id: str | None,
-        operation_type: OperationType,
-        instruction: str,
-        generated_output: str,
-        llm_result: LLMResult | None,
-        model_config: ModelConfig | None = None,
-        timer=None,
-        error: str | None = None,
-    ):
-        if llm_result:
-            prompt_tokens = llm_result.usage.prompt_tokens
-            completion_tokens = llm_result.usage.completion_tokens
-            total_tokens = llm_result.usage.total_tokens
-            model_name = llm_result.model
-            # Extract provider from model_config if available, otherwise fall back to parsing model name
-            if model_config and model_config.provider:
-                model_provider = model_config.provider
-            else:
-                model_provider = model_name.split("/")[0] if "/" in model_name else ""
-            latency = llm_result.usage.latency
-            total_price = float(llm_result.usage.total_price) if llm_result.usage.total_price else None
-            currency = llm_result.usage.currency
-        else:
-            prompt_tokens = 0
-            completion_tokens = 0
-            total_tokens = 0
-            model_provider = model_config.provider if model_config else ""
-            model_name = model_config.name if model_config else ""
-            latency = 0.0
-            if timer:
-                start_time = timer.get("start")
-                end_time = timer.get("end")
-                if start_time and end_time:
-                    latency = (end_time - start_time).total_seconds()
-            total_price = None
-            currency = None
-
-        telemetry_emit(
-            TelemetryEvent(
-                name=TraceTaskName.PROMPT_GENERATION_TRACE,
-                context=TelemetryContext(tenant_id=tenant_id, user_id=user_id, app_id=app_id),
-                payload={
-                    "tenant_id": tenant_id,
-                    "user_id": user_id,
-                    "app_id": app_id,
-                    "operation_type": operation_type,
-                    "instruction": instruction,
-                    "generated_output": generated_output,
-                    "prompt_tokens": prompt_tokens,
-                    "completion_tokens": completion_tokens,
-                    "total_tokens": total_tokens,
-                    "model_provider": model_provider,
-                    "model_name": model_name,
-                    "latency": latency,
-                    "total_price": total_price,
-                    "currency": currency,
-                    "timer": timer,
-                    "error": error,
-                },
-            )
-        )
+            generated_raw = response.message.get_text_content()
+            first_brace = generated_raw.find("{")
+            last_brace = generated_raw.rfind("}")
+            if first_brace == -1 or last_brace == -1 or last_brace < first_brace:
+                raise ValueError(f"Could not find a valid JSON object in response: {generated_raw}")
+            json_str = generated_raw[first_brace : last_brace + 1]
+            data = json_repair.loads(json_str)
+            if not isinstance(data, dict):
+                raise TypeError(f"Expected a JSON object, but got {type(data).__name__}")
+            return data
+        except InvokeError as e:
+            error = str(e)
+            return {"error": f"Failed to generate code. Error: {error}"}
+        except Exception as e:
+            logger.exception("Failed to invoke LLM model, model: %s", json.dumps(model_config.name), exc_info=True)
+            return {"error": f"An unexpected error occurred: {str(e)}"}

api/core/llm_generator/prompts.py

@@ -441,13 +441,11 @@ DEFAULT_GENERATOR_SUMMARY_PROMPT = (
 
 Requirements:
 1. Write a concise summary in plain text
-2. You must write in {language}. No language other than {language} should be used.
+2. Use the same language as the input content
 3. Focus on important facts, concepts, and details
 4. If images are included, describe their key information
 5. Do not use words like "好的", "ok", "I understand", "This text discusses", "The content mentions"
 6. Write directly without extra words
-7. If there is not enough content to generate a meaningful summary, 
-   return an empty string without any explanation or prompt
 
 Output only the summary text. Start summarizing now:
 

api/core/logging/filters.py

@@ -15,23 +15,16 @@ class TraceContextFilter(logging.Filter):
     """
 
     def filter(self, record: logging.LogRecord) -> bool:
-        # Preserve explicit trace_id set by the caller (e.g. emit_metric_only_event)
-        existing_trace_id = getattr(record, "trace_id", "")
-        if not existing_trace_id:
-            # Get trace context from OpenTelemetry
-            trace_id, span_id = self._get_otel_context()
+        # Get trace context from OpenTelemetry
+        trace_id, span_id = self._get_otel_context()
 
-            # Set trace_id (fallback to ContextVar if no OTEL context)
-            if trace_id:
-                record.trace_id = trace_id
-            else:
-                record.trace_id = get_trace_id()
-
-            record.span_id = span_id or ""
+        # Set trace_id (fallback to ContextVar if no OTEL context)
+        if trace_id:
+            record.trace_id = trace_id
         else:
-            # Keep existing trace_id; only fill span_id if missing
-            if not getattr(record, "span_id", ""):
-                record.span_id = ""
+            record.trace_id = get_trace_id()
+
+        record.span_id = span_id or ""
 
         # For backward compatibility, also set req_id
         record.req_id = get_request_id()
@@ -62,12 +55,9 @@ class IdentityContextFilter(logging.Filter):
 
     def filter(self, record: logging.LogRecord) -> bool:
         identity = self._extract_identity()
-        if not getattr(record, "tenant_id", ""):
-            record.tenant_id = identity.get("tenant_id", "")
-        if not getattr(record, "user_id", ""):
-            record.user_id = identity.get("user_id", "")
-        if not getattr(record, "user_type", ""):
-            record.user_type = identity.get("user_type", "")
+        record.tenant_id = identity.get("tenant_id", "")
+        record.user_id = identity.get("user_id", "")
+        record.user_type = identity.get("user_type", "")
         return True
 
     def _extract_identity(self) -> dict[str, str]:

api/core/moderation/input_moderation.py

@@ -5,10 +5,9 @@ from typing import Any
 from core.app.app_config.entities import AppConfig
 from core.moderation.base import ModerationAction, ModerationError
 from core.moderation.factory import ModerationFactory
-from core.ops.ops_trace_manager import TraceQueueManager
+from core.ops.entities.trace_entity import TraceTaskName
+from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.ops.utils import measure_time
-from core.telemetry import TelemetryContext, TelemetryEvent, TraceTaskName
-from core.telemetry import emit as telemetry_emit
 
 logger = logging.getLogger(__name__)
 
@@ -50,18 +49,14 @@ class InputModeration:
             moderation_result = moderation_factory.moderation_for_inputs(inputs, query)
 
         if trace_manager:
-            telemetry_emit(
-                TelemetryEvent(
-                    name=TraceTaskName.MODERATION_TRACE,
-                    context=TelemetryContext(tenant_id=tenant_id, app_id=app_id),
-                    payload={
-                        "message_id": message_id,
-                        "moderation_result": moderation_result,
-                        "inputs": inputs,
-                        "timer": timer,
-                    },
-                ),
-                trace_manager=trace_manager,
+            trace_manager.add_trace_task(
+                TraceTask(
+                    TraceTaskName.MODERATION_TRACE,
+                    message_id=message_id,
+                    moderation_result=moderation_result,
+                    inputs=inputs,
+                    timer=timer,
+                )
             )
 
         if not moderation_result.flagged:

api/core/ops/entities/trace_entity.py

@@ -9,8 +9,8 @@ from pydantic import BaseModel, ConfigDict, field_serializer, field_validator
 class BaseTraceInfo(BaseModel):
     message_id: str | None = None
     message_data: Any | None = None
-    inputs: Union[str, dict[str, Any], list[Any]] | None = None
-    outputs: Union[str, dict[str, Any], list[Any]] | None = None
+    inputs: Union[str, dict[str, Any], list] | None = None
+    outputs: Union[str, dict[str, Any], list] | None = None
     start_time: datetime | None = None
     end_time: datetime | None = None
     metadata: dict[str, Any]
@@ -18,7 +18,7 @@ class BaseTraceInfo(BaseModel):
 
     @field_validator("inputs", "outputs")
     @classmethod
-    def ensure_type(cls, v: str | dict[str, Any] | list[Any] | None) -> str | dict[str, Any] | list[Any] | None:
+    def ensure_type(cls, v):
         if v is None:
             return None
         if isinstance(v, str | dict | list):
@@ -27,48 +27,6 @@ class BaseTraceInfo(BaseModel):
 
     model_config = ConfigDict(protected_namespaces=())
 
-    @property
-    def resolved_trace_id(self) -> str | None:
-        """Get trace_id with intelligent fallback.
-
-        Priority:
-        1. External trace_id (from X-Trace-Id header)
-        2. workflow_run_id (if this trace type has it)
-        3. message_id (as final fallback)
-        """
-        if self.trace_id:
-            return self.trace_id
-
-        # Try workflow_run_id (only exists on workflow-related traces)
-        workflow_run_id = getattr(self, "workflow_run_id", None)
-        if workflow_run_id:
-            return workflow_run_id
-
-        # Final fallback to message_id
-        return str(self.message_id) if self.message_id else None
-
-    @property
-    def resolved_parent_context(self) -> tuple[str | None, str | None]:
-        """Resolve cross-workflow parent linking from metadata.
-
-        Extracts typed parent IDs from the untyped ``parent_trace_context``
-        metadata dict (set by tool_node when invoking nested workflows).
-
-        Returns:
-            (trace_correlation_override, parent_span_id_source) where
-            trace_correlation_override is the outer workflow_run_id and
-            parent_span_id_source is the outer node_execution_id.
-        """
-        parent_ctx = self.metadata.get("parent_trace_context")
-        if not isinstance(parent_ctx, dict):
-            return None, None
-        trace_override = parent_ctx.get("parent_workflow_run_id")
-        parent_span = parent_ctx.get("parent_node_execution_id")
-        return (
-            trace_override if isinstance(trace_override, str) else None,
-            parent_span if isinstance(parent_span, str) else None,
-        )
-
     @field_serializer("start_time", "end_time")
     def serialize_datetime(self, dt: datetime | None) -> str | None:
         if dt is None:
@@ -90,14 +48,10 @@ class WorkflowTraceInfo(BaseTraceInfo):
     workflow_run_version: str
     error: str | None = None
     total_tokens: int
-    prompt_tokens: int | None = None
-    completion_tokens: int | None = None
     file_list: list[str]
     query: str
     metadata: dict[str, Any]
 
-    invoked_by: str | None = None
-
 
 class MessageTraceInfo(BaseTraceInfo):
     conversation_model: str
@@ -105,7 +59,7 @@ class MessageTraceInfo(BaseTraceInfo):
     answer_tokens: int
     total_tokens: int
     error: str | None = None
-    file_list: Union[str, dict[str, Any], list[Any]] | None = None
+    file_list: Union[str, dict[str, Any], list] | None = None
     message_file_data: Any | None = None
     conversation_mode: str
     gen_ai_server_time_to_first_token: float | None = None
@@ -152,7 +106,7 @@ class ToolTraceInfo(BaseTraceInfo):
     tool_config: dict[str, Any]
     time_cost: Union[int, float]
     tool_parameters: dict[str, Any]
-    file_url: Union[str, None, list[str]] = None
+    file_url: Union[str, None, list] = None
 
 
 class GenerateNameTraceInfo(BaseTraceInfo):
@@ -160,79 +114,6 @@ class GenerateNameTraceInfo(BaseTraceInfo):
     tenant_id: str
 
 
-class PromptGenerationTraceInfo(BaseTraceInfo):
-    """Trace information for prompt generation operations (rule-generate, code-generate, etc.)."""
-
-    tenant_id: str
-    user_id: str
-    app_id: str | None = None
-
-    operation_type: str
-    instruction: str
-
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-    model_provider: str
-    model_name: str
-
-    latency: float
-
-    total_price: float | None = None
-    currency: str | None = None
-
-    error: str | None = None
-
-    model_config = ConfigDict(protected_namespaces=())
-
-
-class WorkflowNodeTraceInfo(BaseTraceInfo):
-    workflow_id: str
-    workflow_run_id: str
-    tenant_id: str
-    node_execution_id: str
-    node_id: str
-    node_type: str
-    title: str
-
-    status: str
-    error: str | None = None
-    elapsed_time: float
-
-    index: int
-    predecessor_node_id: str | None = None
-
-    total_tokens: int = 0
-    total_price: float = 0.0
-    currency: str | None = None
-
-    model_provider: str | None = None
-    model_name: str | None = None
-    prompt_tokens: int | None = None
-    completion_tokens: int | None = None
-
-    tool_name: str | None = None
-
-    iteration_id: str | None = None
-    iteration_index: int | None = None
-    loop_id: str | None = None
-    loop_index: int | None = None
-    parallel_id: str | None = None
-
-    node_inputs: Mapping[str, Any] | None = None
-    node_outputs: Mapping[str, Any] | None = None
-    process_data: Mapping[str, Any] | None = None
-
-    invoked_by: str | None = None
-
-    model_config = ConfigDict(protected_namespaces=())
-
-
-class DraftNodeExecutionTrace(WorkflowNodeTraceInfo):
-    pass
-
-
 class TaskData(BaseModel):
     app_id: str
     trace_info_type: str
@@ -247,38 +128,16 @@ trace_info_info_map = {
     "DatasetRetrievalTraceInfo": DatasetRetrievalTraceInfo,
     "ToolTraceInfo": ToolTraceInfo,
     "GenerateNameTraceInfo": GenerateNameTraceInfo,
-    "PromptGenerationTraceInfo": PromptGenerationTraceInfo,
-    "WorkflowNodeTraceInfo": WorkflowNodeTraceInfo,
-    "DraftNodeExecutionTrace": DraftNodeExecutionTrace,
 }
 
 
-class OperationType(StrEnum):
-    """Operation type for token metric labels.
-
-    Used as a metric attribute on ``dify.tokens.input`` / ``dify.tokens.output``
-    counters so consumers can break down token usage by operation.
-    """
-
-    WORKFLOW = "workflow"
-    NODE_EXECUTION = "node_execution"
-    MESSAGE = "message"
-    RULE_GENERATE = "rule_generate"
-    CODE_GENERATE = "code_generate"
-    STRUCTURED_OUTPUT = "structured_output"
-    INSTRUCTION_MODIFY = "instruction_modify"
-
-
 class TraceTaskName(StrEnum):
     CONVERSATION_TRACE = "conversation"
     WORKFLOW_TRACE = "workflow"
-    DRAFT_NODE_EXECUTION_TRACE = "draft_node_execution"
     MESSAGE_TRACE = "message"
     MODERATION_TRACE = "moderation"
     SUGGESTED_QUESTION_TRACE = "suggested_question"
     DATASET_RETRIEVAL_TRACE = "dataset_retrieval"
     TOOL_TRACE = "tool"
     GENERATE_NAME_TRACE = "generate_conversation_name"
-    PROMPT_GENERATION_TRACE = "prompt_generation"
     DATASOURCE_TRACE = "datasource"
-    NODE_EXECUTION_TRACE = "node_execution"

api/core/ops/langfuse_trace/langfuse_trace.py

@@ -3,7 +3,6 @@ import os
 from datetime import datetime, timedelta
 
 from langfuse import Langfuse
-from sqlalchemy import select
 from sqlalchemy.orm import sessionmaker
 
 from core.ops.base_trace_instance import BaseTraceInstance
@@ -31,7 +30,7 @@ from core.ops.utils import filter_none_values
 from core.repositories import DifyCoreRepositoryFactory
 from core.workflow.enums import NodeType
 from extensions.ext_database import db
-from models import EndUser, Message, WorkflowNodeExecutionTriggeredFrom
+from models import EndUser, WorkflowNodeExecutionTriggeredFrom
 from models.enums import MessageStatus
 
 logger = logging.getLogger(__name__)
@@ -72,50 +71,7 @@ class LangFuseDataTrace(BaseTraceInstance):
         metadata = trace_info.metadata
         metadata["workflow_app_log_id"] = trace_info.workflow_app_log_id
 
-        # Check for parent_trace_context to detect nested workflow
-        parent_trace_context = trace_info.metadata.get("parent_trace_context")
-
-        if parent_trace_context:
-            # Nested workflow: create span under outer trace
-            outer_trace_id = parent_trace_context.get("trace_id")
-            parent_node_execution_id = parent_trace_context.get("parent_node_execution_id")
-            parent_conversation_id = parent_trace_context.get("parent_conversation_id")
-            parent_workflow_run_id = parent_trace_context.get("parent_workflow_run_id")
-
-            # Resolve outer trace_id: try message_id lookup first, fallback to workflow_run_id
-            if parent_conversation_id:
-                session_factory = sessionmaker(bind=db.engine)
-                with session_factory() as session:
-                    message_data_stmt = select(Message.id).where(
-                        Message.conversation_id == parent_conversation_id,
-                        Message.workflow_run_id == parent_workflow_run_id,
-                    )
-                    resolved_message_id = session.scalar(message_data_stmt)
-                    if resolved_message_id:
-                        outer_trace_id = resolved_message_id
-                    else:
-                        outer_trace_id = parent_workflow_run_id
-            else:
-                outer_trace_id = parent_workflow_run_id
-
-            # Create inner workflow span under outer trace
-            workflow_span_data = LangfuseSpan(
-                id=trace_info.workflow_run_id,
-                name=TraceTaskName.WORKFLOW_TRACE,
-                input=dict(trace_info.workflow_run_inputs),
-                output=dict(trace_info.workflow_run_outputs),
-                trace_id=outer_trace_id,
-                parent_observation_id=parent_node_execution_id,
-                start_time=trace_info.start_time,
-                end_time=trace_info.end_time,
-                metadata=metadata,
-                level=LevelEnum.DEFAULT if trace_info.error == "" else LevelEnum.ERROR,
-                status_message=trace_info.error or "",
-            )
-            self.add_span(langfuse_span_data=workflow_span_data)
-            # Use outer_trace_id for all node spans/generations
-            trace_id = outer_trace_id
-        elif trace_info.message_id:
+        if trace_info.message_id:
             trace_id = trace_info.trace_id or trace_info.message_id
             name = TraceTaskName.MESSAGE_TRACE
             trace_data = LangfuseTrace(
@@ -218,11 +174,6 @@ class LangFuseDataTrace(BaseTraceInstance):
                     }
                 )
 
-            # Determine parent_observation_id for nested workflows
-            node_parent_observation_id = None
-            if parent_trace_context or trace_info.message_id:
-                node_parent_observation_id = trace_info.workflow_run_id
-
             # add generation span
             if process_data and process_data.get("model_mode") == "chat":
                 total_token = metadata.get("total_tokens", 0)
@@ -255,7 +206,7 @@ class LangFuseDataTrace(BaseTraceInstance):
                     metadata=metadata,
                     level=(LevelEnum.DEFAULT if status == "succeeded" else LevelEnum.ERROR),
                     status_message=trace_info.error or "",
-                    parent_observation_id=node_parent_observation_id,
+                    parent_observation_id=trace_info.workflow_run_id if trace_info.message_id else None,
                     usage=generation_usage,
                 )
 
@@ -274,7 +225,7 @@ class LangFuseDataTrace(BaseTraceInstance):
                     metadata=metadata,
                     level=(LevelEnum.DEFAULT if status == "succeeded" else LevelEnum.ERROR),
                     status_message=trace_info.error or "",
-                    parent_observation_id=node_parent_observation_id,
+                    parent_observation_id=trace_info.workflow_run_id if trace_info.message_id else None,
                 )
 
                 self.add_span(langfuse_span_data=span_data)

api/core/ops/langsmith_trace/langsmith_trace.py

@@ -6,7 +6,6 @@ from typing import cast
 
 from langsmith import Client
 from langsmith.schemas import RunBase
-from sqlalchemy import select
 from sqlalchemy.orm import sessionmaker
 
 from core.ops.base_trace_instance import BaseTraceInstance
@@ -31,7 +30,7 @@ from core.ops.utils import filter_none_values, generate_dotted_order
 from core.repositories import DifyCoreRepositoryFactory
 from core.workflow.enums import NodeType, WorkflowNodeExecutionMetadataKey
 from extensions.ext_database import db
-from models import EndUser, Message, MessageFile, WorkflowNodeExecutionTriggeredFrom
+from models import EndUser, MessageFile, WorkflowNodeExecutionTriggeredFrom
 
 logger = logging.getLogger(__name__)
 
@@ -65,35 +64,7 @@ class LangSmithDataTrace(BaseTraceInstance):
             self.generate_name_trace(trace_info)
 
     def workflow_trace(self, trace_info: WorkflowTraceInfo):
-        # Check for parent_trace_context for cross-workflow linking
-        parent_trace_context = trace_info.metadata.get("parent_trace_context")
-
-        if parent_trace_context:
-            # Inner workflow: resolve outer trace_id and link to parent node
-            outer_trace_id = parent_trace_context.get("parent_workflow_run_id")
-
-            # Try to resolve message_id from conversation_id if available
-            if parent_trace_context.get("parent_conversation_id"):
-                try:
-                    session_factory = sessionmaker(bind=db.engine)
-                    with session_factory() as session:
-                        message_data_stmt = select(Message.id).where(
-                            Message.conversation_id == parent_trace_context["parent_conversation_id"],
-                            Message.workflow_run_id == parent_trace_context["parent_workflow_run_id"],
-                        )
-                        resolved_message_id = session.scalar(message_data_stmt)
-                        if resolved_message_id:
-                            outer_trace_id = resolved_message_id
-                except Exception as e:
-                    logger.debug("Failed to resolve message_id from conversation_id: %s", str(e))
-
-            trace_id = outer_trace_id
-            parent_run_id = parent_trace_context.get("parent_node_execution_id")
-        else:
-            # Outer workflow: existing behavior
-            trace_id = trace_info.trace_id or trace_info.message_id or trace_info.workflow_run_id
-            parent_run_id = trace_info.message_id or None
-
+        trace_id = trace_info.trace_id or trace_info.message_id or trace_info.workflow_run_id
         if trace_info.start_time is None:
             trace_info.start_time = datetime.now()
         message_dotted_order = (
@@ -107,8 +78,7 @@ class LangSmithDataTrace(BaseTraceInstance):
         metadata = trace_info.metadata
         metadata["workflow_app_log_id"] = trace_info.workflow_app_log_id
 
-        # Only create message_run for outer workflows (no parent_trace_context)
-        if trace_info.message_id and not parent_trace_context:
+        if trace_info.message_id:
             message_run = LangSmithRunModel(
                 id=trace_info.message_id,
                 name=TraceTaskName.MESSAGE_TRACE,
@@ -151,9 +121,9 @@ class LangSmithDataTrace(BaseTraceInstance):
             },
             error=trace_info.error,
             tags=["workflow"],
-            parent_run_id=parent_run_id,
+            parent_run_id=trace_info.message_id or None,
             trace_id=trace_id,
-            dotted_order=None if parent_trace_context else workflow_dotted_order,
+            dotted_order=workflow_dotted_order,
             serialized=None,
             events=[],
             session_id=None,

api/core/ops/ops_trace_manager.py

@@ -21,26 +21,19 @@ from core.ops.entities.config_entity import (
 )
 from core.ops.entities.trace_entity import (
     DatasetRetrievalTraceInfo,
-    DraftNodeExecutionTrace,
     GenerateNameTraceInfo,
     MessageTraceInfo,
     ModerationTraceInfo,
-    PromptGenerationTraceInfo,
     SuggestedQuestionTraceInfo,
     TaskData,
     ToolTraceInfo,
     TraceTaskName,
-    WorkflowNodeTraceInfo,
     WorkflowTraceInfo,
 )
 from core.ops.utils import get_message_data
 from extensions.ext_database import db
 from extensions.ext_storage import storage
-from models.account import Tenant
-from models.dataset import Dataset
 from models.model import App, AppModelConfig, Conversation, Message, MessageFile, TraceAppConfig
-from models.provider import Provider, ProviderCredential, ProviderModel, ProviderModelCredential, ProviderType
-from models.tools import ApiToolProvider, BuiltinToolProvider, MCPToolProvider, WorkflowToolProvider
 from models.workflow import WorkflowAppLog
 from tasks.ops_trace_task import process_trace_tasks
 
@@ -50,139 +43,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _lookup_app_and_workspace_names(app_id: str | None, tenant_id: str | None) -> tuple[str, str]:
-    """Return (app_name, workspace_name) for the given IDs. Falls back to empty strings."""
-    app_name = ""
-    workspace_name = ""
-    if not app_id and not tenant_id:
-        return app_name, workspace_name
-    with Session(db.engine) as session:
-        if app_id:
-            name = session.scalar(select(App.name).where(App.id == app_id))
-            if name:
-                app_name = name
-        if tenant_id:
-            name = session.scalar(select(Tenant.name).where(Tenant.id == tenant_id))
-            if name:
-                workspace_name = name
-    return app_name, workspace_name
-
-
-_PROVIDER_TYPE_TO_MODEL: dict[str, type] = {
-    "builtin": BuiltinToolProvider,
-    "plugin": BuiltinToolProvider,
-    "api": ApiToolProvider,
-    "workflow": WorkflowToolProvider,
-    "mcp": MCPToolProvider,
-}
-
-
-def _lookup_credential_name(credential_id: str | None, provider_type: str | None) -> str:
-    if not credential_id:
-        return ""
-    model_cls = _PROVIDER_TYPE_TO_MODEL.get(provider_type or "")
-    if not model_cls:
-        return ""
-    with Session(db.engine) as session:
-        name = session.scalar(select(model_cls.name).where(model_cls.id == credential_id))
-        return str(name) if name else ""
-
-
-def _lookup_llm_credential_info(
-    tenant_id: str | None, provider: str | None, model: str | None, model_type: str | None = "llm"
-) -> tuple[str | None, str]:
-    """
-    Lookup LLM credential ID and name for the given provider and model.
-    Returns (credential_id, credential_name).
-
-    Handles async timing issues gracefully - if credential is deleted between lookups,
-    returns the ID but empty name rather than failing.
-    """
-    if not tenant_id or not provider:
-        return None, ""
-
-    try:
-        with Session(db.engine) as session:
-            # Try to find provider-level or model-level configuration
-            provider_record = session.scalar(
-                select(Provider).where(
-                    Provider.tenant_id == tenant_id,
-                    Provider.provider_name == provider,
-                    Provider.provider_type == ProviderType.CUSTOM,
-                )
-            )
-
-            if not provider_record:
-                return None, ""
-
-            # Check if there's a model-specific config
-            credential_id = None
-            credential_name = ""
-            is_model_level = False
-
-            if model and provider_record.credential_id:
-                # Try model-level first
-                model_record = session.scalar(
-                    select(ProviderModel).where(
-                        ProviderModel.tenant_id == tenant_id,
-                        ProviderModel.provider_name == provider,
-                        ProviderModel.model_name == model,
-                        ProviderModel.model_type == model_type,
-                    )
-                )
-
-                if model_record and model_record.credential_id:
-                    credential_id = model_record.credential_id
-                    is_model_level = True
-
-            if not credential_id and provider_record.credential_id:
-                # Fall back to provider-level credential
-                credential_id = provider_record.credential_id
-                is_model_level = False
-
-            # Lookup credential_name if we have credential_id
-            if credential_id:
-                try:
-                    if is_model_level:
-                        # Query ProviderModelCredential
-                        cred_name = session.scalar(
-                            select(ProviderModelCredential.credential_name).where(
-                                ProviderModelCredential.id == credential_id
-                            )
-                        )
-                    else:
-                        # Query ProviderCredential
-                        cred_name = session.scalar(
-                            select(ProviderCredential.credential_name).where(ProviderCredential.id == credential_id)
-                        )
-
-                    if cred_name:
-                        credential_name = str(cred_name)
-                except Exception as e:
-                    # Credential might have been deleted between lookups (async timing)
-                    # Return ID but empty name rather than failing
-                    logger.warning(
-                        "Failed to lookup credential name for credential_id=%s (provider=%s, model=%s): %s",
-                        credential_id,
-                        provider,
-                        model,
-                        str(e),
-                    )
-
-            return credential_id, credential_name
-    except Exception as e:
-        # Database query failed or other unexpected error
-        # Return empty rather than propagating error to telemetry emission
-        logger.warning(
-            "Failed to lookup LLM credential info for tenant_id=%s, provider=%s, model=%s: %s",
-            tenant_id,
-            provider,
-            model,
-            str(e),
-        )
-        return None, ""
-
-
 class OpsTraceProviderConfigMap(collections.UserDict[str, dict[str, Any]]):
     def __getitem__(self, provider: str) -> dict[str, Any]:
         match provider:
@@ -457,10 +317,6 @@ class OpsTraceManager:
         if app_id is None:
             return None
 
-        # Handle storage_id format (tenant-{uuid}) - not a real app_id
-        if isinstance(app_id, str) and app_id.startswith("tenant-"):
-            return None
-
         app: App | None = db.session.query(App).where(App.id == app_id).first()
 
         if app is None:
@@ -623,56 +479,6 @@ class TraceTask:
                     cls._workflow_run_repo = DifyAPIRepositoryFactory.create_api_workflow_run_repository(session_maker)
         return cls._workflow_run_repo
 
-    @classmethod
-    def _get_user_id_from_metadata(cls, metadata: dict[str, Any]) -> str:
-        """Extract user ID from metadata, prioritizing end_user over account.
-
-        Returns the actual user ID (end_user or account) who invoked the workflow,
-        regardless of invoke_from context.
-        """
-        # Priority 1: End user (external users via API/WebApp)
-        if user_id := metadata.get("from_end_user_id"):
-            return f"end_user:{user_id}"
-
-        # Priority 2: Account user (internal users via console/debugger)
-        if user_id := metadata.get("from_account_id"):
-            return f"account:{user_id}"
-
-        # Priority 3: User (internal users via console/debugger)
-        if user_id := metadata.get("user_id"):
-            return f"user:{user_id}"
-
-        return "anonymous"
-
-    @classmethod
-    def _calculate_workflow_token_split(cls, workflow_run_id: str, tenant_id: str) -> tuple[int, int]:
-        from core.workflow.enums import WorkflowNodeExecutionMetadataKey
-        from models.workflow import WorkflowNodeExecutionModel
-
-        with Session(db.engine) as session:
-            node_executions = session.scalars(
-                select(WorkflowNodeExecutionModel).where(
-                    WorkflowNodeExecutionModel.tenant_id == tenant_id,
-                    WorkflowNodeExecutionModel.workflow_run_id == workflow_run_id,
-                )
-            ).all()
-
-            total_prompt = 0
-            total_completion = 0
-
-            for node_exec in node_executions:
-                metadata = node_exec.execution_metadata_dict
-
-                prompt = metadata.get(WorkflowNodeExecutionMetadataKey.PROMPT_TOKENS)
-                if prompt is not None:
-                    total_prompt += prompt
-
-                completion = metadata.get(WorkflowNodeExecutionMetadataKey.COMPLETION_TOKENS)
-                if completion is not None:
-                    total_completion += completion
-
-            return (total_prompt, total_completion)
-
     def __init__(
         self,
         trace_type: Any,
@@ -693,8 +499,6 @@ class TraceTask:
         self.app_id = None
         self.trace_id = None
         self.kwargs = kwargs
-        if user_id is not None and "user_id" not in self.kwargs:
-            self.kwargs["user_id"] = user_id
         external_trace_id = kwargs.get("external_trace_id")
         if external_trace_id:
             self.trace_id = external_trace_id
@@ -708,7 +512,7 @@ class TraceTask:
             TraceTaskName.WORKFLOW_TRACE: lambda: self.workflow_trace(
                 workflow_run_id=self.workflow_run_id, conversation_id=self.conversation_id, user_id=self.user_id
             ),
-            TraceTaskName.MESSAGE_TRACE: lambda: self.message_trace(message_id=self.message_id, **self.kwargs),
+            TraceTaskName.MESSAGE_TRACE: lambda: self.message_trace(message_id=self.message_id),
             TraceTaskName.MODERATION_TRACE: lambda: self.moderation_trace(
                 message_id=self.message_id, timer=self.timer, **self.kwargs
             ),
@@ -724,9 +528,6 @@ class TraceTask:
             TraceTaskName.GENERATE_NAME_TRACE: lambda: self.generate_name_trace(
                 conversation_id=self.conversation_id, timer=self.timer, **self.kwargs
             ),
-            TraceTaskName.PROMPT_GENERATION_TRACE: lambda: self.prompt_generation_trace(**self.kwargs),
-            TraceTaskName.NODE_EXECUTION_TRACE: lambda: self.node_execution_trace(**self.kwargs),
-            TraceTaskName.DRAFT_NODE_EXECUTION_TRACE: lambda: self.draft_node_execution_trace(**self.kwargs),
         }
 
         return preprocess_map.get(self.trace_type, lambda: None)()
@@ -762,10 +563,6 @@ class TraceTask:
 
         total_tokens = workflow_run.total_tokens
 
-        prompt_tokens, completion_tokens = self._calculate_workflow_token_split(
-            workflow_run_id=workflow_run_id, tenant_id=tenant_id
-        )
-
         file_list = workflow_run_inputs.get("sys.file") or []
         query = workflow_run_inputs.get("query") or workflow_run_inputs.get("sys.query") or ""
 
@@ -786,14 +583,7 @@ class TraceTask:
                 )
                 message_id = session.scalar(message_data_stmt)
 
-        from core.telemetry.gateway import is_enterprise_telemetry_enabled
-
-        if is_enterprise_telemetry_enabled():
-            app_name, workspace_name = _lookup_app_and_workspace_names(workflow_run.app_id, tenant_id)
-        else:
-            app_name, workspace_name = "", ""
-
-        metadata: dict[str, Any] = {
+        metadata = {
             "workflow_id": workflow_id,
             "conversation_id": conversation_id,
             "workflow_run_id": workflow_run_id,
@@ -806,14 +596,8 @@ class TraceTask:
             "triggered_from": workflow_run.triggered_from,
             "user_id": user_id,
             "app_id": workflow_run.app_id,
-            "app_name": app_name,
-            "workspace_name": workspace_name,
         }
 
-        parent_trace_context = self.kwargs.get("parent_trace_context")
-        if parent_trace_context:
-            metadata["parent_trace_context"] = parent_trace_context
-
         workflow_trace_info = WorkflowTraceInfo(
             trace_id=self.trace_id,
             workflow_data=workflow_run.to_dict(),
@@ -828,8 +612,6 @@ class TraceTask:
             workflow_run_version=workflow_run_version,
             error=error,
             total_tokens=total_tokens,
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
             file_list=file_list,
             query=query,
             metadata=metadata,
@@ -837,11 +619,10 @@ class TraceTask:
             message_id=message_id,
             start_time=workflow_run.created_at,
             end_time=workflow_run.finished_at,
-            invoked_by=self._get_user_id_from_metadata(metadata),
         )
         return workflow_trace_info
 
-    def message_trace(self, message_id: str | None, **kwargs):
+    def message_trace(self, message_id: str | None):
         if not message_id:
             return {}
         message_data = get_message_data(message_id)
@@ -864,19 +645,6 @@ class TraceTask:
 
         streaming_metrics = self._extract_streaming_metrics(message_data)
 
-        tenant_id = ""
-        with Session(db.engine) as session:
-            tid = session.scalar(select(App.tenant_id).where(App.id == message_data.app_id))
-            if tid:
-                tenant_id = str(tid)
-
-        from core.telemetry.gateway import is_enterprise_telemetry_enabled
-
-        if is_enterprise_telemetry_enabled():
-            app_name, workspace_name = _lookup_app_and_workspace_names(message_data.app_id, tenant_id)
-        else:
-            app_name, workspace_name = "", ""
-
         metadata = {
             "conversation_id": message_data.conversation_id,
             "ls_provider": message_data.model_provider,
@@ -888,14 +656,7 @@ class TraceTask:
             "workflow_run_id": message_data.workflow_run_id,
             "from_source": message_data.from_source,
             "message_id": message_id,
-            "tenant_id": tenant_id,
-            "app_id": message_data.app_id,
-            "user_id": message_data.from_end_user_id or message_data.from_account_id,
-            "app_name": app_name,
-            "workspace_name": workspace_name,
         }
-        if node_execution_id := kwargs.get("node_execution_id"):
-            metadata["node_execution_id"] = node_execution_id
 
         message_tokens = message_data.message_tokens
 
@@ -912,9 +673,7 @@ class TraceTask:
             outputs=message_data.answer,
             file_list=file_list,
             start_time=created_at,
-            end_time=message_data.updated_at
-            if message_data.updated_at and message_data.updated_at > created_at
-            else created_at + timedelta(seconds=message_data.provider_response_latency),
+            end_time=created_at + timedelta(seconds=message_data.provider_response_latency),
             metadata=metadata,
             message_file_data=message_file_data,
             conversation_mode=conversation_mode,
@@ -939,8 +698,6 @@ class TraceTask:
             "preset_response": moderation_result.preset_response,
             "query": moderation_result.query,
         }
-        if node_execution_id := kwargs.get("node_execution_id"):
-            metadata["node_execution_id"] = node_execution_id
 
         # get workflow_app_log_id
         workflow_app_log_id = None
@@ -982,8 +739,6 @@ class TraceTask:
             "workflow_run_id": message_data.workflow_run_id,
             "from_source": message_data.from_source,
         }
-        if node_execution_id := kwargs.get("node_execution_id"):
-            metadata["node_execution_id"] = node_execution_id
 
         # get workflow_app_log_id
         workflow_app_log_id = None
@@ -1023,52 +778,6 @@ class TraceTask:
         if not message_data:
             return {}
 
-        tenant_id = ""
-        with Session(db.engine) as session:
-            tid = session.scalar(select(App.tenant_id).where(App.id == message_data.app_id))
-            if tid:
-                tenant_id = str(tid)
-
-        from core.telemetry.gateway import is_enterprise_telemetry_enabled
-
-        if is_enterprise_telemetry_enabled():
-            app_name, workspace_name = _lookup_app_and_workspace_names(message_data.app_id, tenant_id)
-        else:
-            app_name, workspace_name = "", ""
-
-        doc_list = [doc.model_dump() for doc in documents] if documents else []
-        dataset_ids: set[str] = set()
-        for doc in doc_list:
-            doc_meta = doc.get("metadata") or {}
-            did = doc_meta.get("dataset_id")
-            if did:
-                dataset_ids.add(did)
-
-        embedding_models: dict[str, dict[str, str]] = {}
-        if dataset_ids:
-            with Session(db.engine) as session:
-                rows = session.execute(
-                    select(Dataset.id, Dataset.embedding_model, Dataset.embedding_model_provider).where(
-                        Dataset.id.in_(list(dataset_ids))
-                    )
-                ).all()
-                for row in rows:
-                    embedding_models[str(row[0])] = {
-                        "embedding_model": row[1] or "",
-                        "embedding_model_provider": row[2] or "",
-                    }
-
-        # Extract rerank model info from retrieval_model kwargs
-        rerank_model_provider = ""
-        rerank_model_name = ""
-        if "retrieval_model" in kwargs:
-            retrieval_model = kwargs["retrieval_model"]
-            if isinstance(retrieval_model, dict):
-                reranking_model = retrieval_model.get("reranking_model")
-                if isinstance(reranking_model, dict):
-                    rerank_model_provider = reranking_model.get("reranking_provider_name", "")
-                    rerank_model_name = reranking_model.get("reranking_model_name", "")
-
         metadata = {
             "message_id": message_id,
             "ls_provider": message_data.model_provider,
@@ -1079,23 +788,13 @@ class TraceTask:
             "agent_based": message_data.agent_based,
             "workflow_run_id": message_data.workflow_run_id,
             "from_source": message_data.from_source,
-            "tenant_id": tenant_id,
-            "app_id": message_data.app_id,
-            "user_id": message_data.from_end_user_id or message_data.from_account_id,
-            "app_name": app_name,
-            "workspace_name": workspace_name,
-            "embedding_models": embedding_models,
-            "rerank_model_provider": rerank_model_provider,
-            "rerank_model_name": rerank_model_name,
         }
-        if node_execution_id := kwargs.get("node_execution_id"):
-            metadata["node_execution_id"] = node_execution_id
 
         dataset_retrieval_trace_info = DatasetRetrievalTraceInfo(
             trace_id=self.trace_id,
             message_id=message_id,
             inputs=message_data.query or message_data.inputs,
-            documents=doc_list,
+            documents=[doc.model_dump() for doc in documents] if documents else [],
             start_time=timer.get("start"),
             end_time=timer.get("end"),
             metadata=metadata,
@@ -1138,10 +837,6 @@ class TraceTask:
             "error": error,
             "tool_parameters": tool_parameters,
         }
-        if message_data.workflow_run_id:
-            metadata["workflow_run_id"] = message_data.workflow_run_id
-        if node_execution_id := kwargs.get("node_execution_id"):
-            metadata["node_execution_id"] = node_execution_id
 
         file_url = ""
         message_file_data = db.session.query(MessageFile).filter_by(message_id=message_id).first()
@@ -1196,8 +891,6 @@ class TraceTask:
             "conversation_id": conversation_id,
             "tenant_id": tenant_id,
         }
-        if node_execution_id := kwargs.get("node_execution_id"):
-            metadata["node_execution_id"] = node_execution_id
 
         generate_name_trace_info = GenerateNameTraceInfo(
             trace_id=self.trace_id,
@@ -1212,182 +905,6 @@ class TraceTask:
 
         return generate_name_trace_info
 
-    def prompt_generation_trace(self, **kwargs) -> PromptGenerationTraceInfo | dict:
-        tenant_id = kwargs.get("tenant_id", "")
-        user_id = kwargs.get("user_id", "")
-        app_id = kwargs.get("app_id")
-        operation_type = kwargs.get("operation_type", "")
-        instruction = kwargs.get("instruction", "")
-        generated_output = kwargs.get("generated_output", "")
-
-        prompt_tokens = kwargs.get("prompt_tokens", 0)
-        completion_tokens = kwargs.get("completion_tokens", 0)
-        total_tokens = kwargs.get("total_tokens", 0)
-
-        model_provider = kwargs.get("model_provider", "")
-        model_name = kwargs.get("model_name", "")
-
-        latency = kwargs.get("latency", 0.0)
-
-        timer = kwargs.get("timer")
-        start_time = timer.get("start") if timer else None
-        end_time = timer.get("end") if timer else None
-
-        total_price = kwargs.get("total_price")
-        currency = kwargs.get("currency")
-
-        error = kwargs.get("error")
-
-        app_name = None
-        workspace_name = None
-        if app_id:
-            app_name, workspace_name = _lookup_app_and_workspace_names(app_id, tenant_id)
-
-        metadata = {
-            "tenant_id": tenant_id,
-            "user_id": user_id,
-            "app_id": app_id or "",
-            "app_name": app_name,
-            "workspace_name": workspace_name,
-            "operation_type": operation_type,
-            "model_provider": model_provider,
-            "model_name": model_name,
-        }
-        if node_execution_id := kwargs.get("node_execution_id"):
-            metadata["node_execution_id"] = node_execution_id
-
-        return PromptGenerationTraceInfo(
-            trace_id=self.trace_id,
-            inputs=instruction,
-            outputs=generated_output,
-            start_time=start_time,
-            end_time=end_time,
-            metadata=metadata,
-            tenant_id=tenant_id,
-            user_id=user_id,
-            app_id=app_id,
-            operation_type=operation_type,
-            instruction=instruction,
-            prompt_tokens=prompt_tokens,
-            completion_tokens=completion_tokens,
-            total_tokens=total_tokens,
-            model_provider=model_provider,
-            model_name=model_name,
-            latency=latency,
-            total_price=total_price,
-            currency=currency,
-            error=error,
-        )
-
-    def node_execution_trace(self, **kwargs) -> WorkflowNodeTraceInfo | dict:
-        node_data: dict = kwargs.get("node_execution_data", {})
-        if not node_data:
-            return {}
-
-        from core.telemetry.gateway import is_enterprise_telemetry_enabled
-
-        if is_enterprise_telemetry_enabled():
-            app_name, workspace_name = _lookup_app_and_workspace_names(
-                node_data.get("app_id"), node_data.get("tenant_id")
-            )
-        else:
-            app_name, workspace_name = "", ""
-
-        # Try tool credential lookup first
-        credential_id = node_data.get("credential_id")
-        if is_enterprise_telemetry_enabled():
-            credential_name = _lookup_credential_name(credential_id, node_data.get("credential_provider_type"))
-            # If no credential_id found (e.g., LLM nodes), try LLM credential lookup
-            if not credential_id:
-                llm_cred_id, llm_cred_name = _lookup_llm_credential_info(
-                    tenant_id=node_data.get("tenant_id"),
-                    provider=node_data.get("model_provider"),
-                    model=node_data.get("model_name"),
-                    model_type="llm",
-                )
-                if llm_cred_id:
-                    credential_id = llm_cred_id
-                    credential_name = llm_cred_name
-        else:
-            credential_name = ""
-        metadata: dict[str, Any] = {
-            "tenant_id": node_data.get("tenant_id"),
-            "app_id": node_data.get("app_id"),
-            "app_name": app_name,
-            "workspace_name": workspace_name,
-            "user_id": node_data.get("user_id"),
-            "invoke_from": node_data.get("invoke_from"),
-            "credential_id": node_data.get("credential_id"),
-            "credential_name": credential_name,
-            "dataset_ids": node_data.get("dataset_ids"),
-            "dataset_names": node_data.get("dataset_names"),
-            "plugin_name": node_data.get("plugin_name"),
-        }
-
-        parent_trace_context = node_data.get("parent_trace_context")
-        if parent_trace_context:
-            metadata["parent_trace_context"] = parent_trace_context
-
-        message_id: str | None = None
-        conversation_id = node_data.get("conversation_id")
-        workflow_execution_id = node_data.get("workflow_execution_id")
-        if conversation_id and workflow_execution_id and not parent_trace_context:
-            with Session(db.engine) as session:
-                msg_id = session.scalar(
-                    select(Message.id).where(
-                        Message.conversation_id == conversation_id,
-                        Message.workflow_run_id == workflow_execution_id,
-                    )
-                )
-                if msg_id:
-                    message_id = str(msg_id)
-                    metadata["message_id"] = message_id
-            if conversation_id:
-                metadata["conversation_id"] = conversation_id
-
-        return WorkflowNodeTraceInfo(
-            trace_id=self.trace_id,
-            message_id=message_id,
-            start_time=node_data.get("created_at"),
-            end_time=node_data.get("finished_at"),
-            metadata=metadata,
-            workflow_id=node_data.get("workflow_id", ""),
-            workflow_run_id=node_data.get("workflow_execution_id", ""),
-            tenant_id=node_data.get("tenant_id", ""),
-            node_execution_id=node_data.get("node_execution_id", ""),
-            node_id=node_data.get("node_id", ""),
-            node_type=node_data.get("node_type", ""),
-            title=node_data.get("title", ""),
-            status=node_data.get("status", ""),
-            error=node_data.get("error"),
-            elapsed_time=node_data.get("elapsed_time", 0.0),
-            index=node_data.get("index", 0),
-            predecessor_node_id=node_data.get("predecessor_node_id"),
-            total_tokens=node_data.get("total_tokens", 0),
-            total_price=node_data.get("total_price", 0.0),
-            currency=node_data.get("currency"),
-            model_provider=node_data.get("model_provider"),
-            model_name=node_data.get("model_name"),
-            prompt_tokens=node_data.get("prompt_tokens"),
-            completion_tokens=node_data.get("completion_tokens"),
-            tool_name=node_data.get("tool_name"),
-            iteration_id=node_data.get("iteration_id"),
-            iteration_index=node_data.get("iteration_index"),
-            loop_id=node_data.get("loop_id"),
-            loop_index=node_data.get("loop_index"),
-            parallel_id=node_data.get("parallel_id"),
-            node_inputs=node_data.get("node_inputs"),
-            node_outputs=node_data.get("node_outputs"),
-            process_data=node_data.get("process_data"),
-            invoked_by=self._get_user_id_from_metadata(metadata),
-        )
-
-    def draft_node_execution_trace(self, **kwargs) -> DraftNodeExecutionTrace | dict:
-        node_trace = self.node_execution_trace(**kwargs)
-        if not node_trace or not isinstance(node_trace, WorkflowNodeTraceInfo):
-            return node_trace
-        return DraftNodeExecutionTrace(**node_trace.model_dump())
-
     def _extract_streaming_metrics(self, message_data) -> dict:
         if not message_data.message_metadata:
             return {}
@@ -1421,17 +938,13 @@ class TraceQueueManager:
         self.user_id = user_id
         self.trace_instance = OpsTraceManager.get_ops_trace_instance(app_id)
         self.flask_app = current_app._get_current_object()  # type: ignore
-
-        from core.telemetry.gateway import is_enterprise_telemetry_enabled
-
-        self._enterprise_telemetry_enabled = is_enterprise_telemetry_enabled()
         if trace_manager_timer is None:
             self.start_timer()
 
     def add_trace_task(self, trace_task: TraceTask):
         global trace_manager_timer, trace_manager_queue
         try:
-            if self._enterprise_telemetry_enabled or self.trace_instance:
+            if self.trace_instance:
                 trace_task.app_id = self.app_id
                 trace_manager_queue.put(trace_task)
         except Exception:
@@ -1467,27 +980,20 @@ class TraceQueueManager:
     def send_to_celery(self, tasks: list[TraceTask]):
         with self.flask_app.app_context():
             for task in tasks:
-                storage_id = task.app_id
-                if storage_id is None:
-                    tenant_id = task.kwargs.get("tenant_id")
-                    if tenant_id:
-                        storage_id = f"tenant-{tenant_id}"
-                    else:
-                        logger.warning("Skipping trace without app_id or tenant_id, trace_type: %s", task.trace_type)
-                        continue
-
+                if task.app_id is None:
+                    continue
                 file_id = uuid4().hex
                 trace_info = task.execute()
 
                 task_data = TaskData(
-                    app_id=storage_id,
+                    app_id=task.app_id,
                     trace_info_type=type(trace_info).__name__,
                     trace_info=trace_info.model_dump() if trace_info else None,
                 )
-                file_path = f"{OPS_FILE_PATH}{storage_id}/{file_id}.json"
+                file_path = f"{OPS_FILE_PATH}{task.app_id}/{file_id}.json"
                 storage.save(file_path, task_data.model_dump_json().encode("utf-8"))
                 file_info = {
                     "file_id": file_id,
-                    "app_id": storage_id,
+                    "app_id": task.app_id,
                 }
                 process_trace_tasks.delay(file_info)  # type: ignore

api/core/rag/index_processor/index_processor_base.py

@@ -48,22 +48,12 @@ class BaseIndexProcessor(ABC):
 
     @abstractmethod
     def generate_summary_preview(
-        self,
-        tenant_id: str,
-        preview_texts: list[PreviewDetail],
-        summary_index_setting: dict,
-        doc_language: str | None = None,
+        self, tenant_id: str, preview_texts: list[PreviewDetail], summary_index_setting: dict
     ) -> list[PreviewDetail]:
         """
         For each segment in preview_texts, generate a summary using LLM and attach it to the segment.
         The summary can be stored in a new attribute, e.g., summary.
         This method should be implemented by subclasses.
-
-        Args:
-            tenant_id: Tenant ID
-            preview_texts: List of preview details to generate summaries for
-            summary_index_setting: Summary index configuration
-            doc_language: Optional document language to ensure summary is generated in the correct language
         """
         raise NotImplementedError
 

api/core/rag/index_processor/processor/paragraph_index_processor.py

@@ -275,11 +275,7 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
             raise ValueError("Chunks is not a list")
 
     def generate_summary_preview(
-        self,
-        tenant_id: str,
-        preview_texts: list[PreviewDetail],
-        summary_index_setting: dict,
-        doc_language: str | None = None,
+        self, tenant_id: str, preview_texts: list[PreviewDetail], summary_index_setting: dict
     ) -> list[PreviewDetail]:
         """
         For each segment, concurrently call generate_summary to generate a summary
@@ -302,15 +298,11 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
             if flask_app:
                 # Ensure Flask app context in worker thread
                 with flask_app.app_context():
-                    summary, _ = self.generate_summary(
-                        tenant_id, preview.content, summary_index_setting, document_language=doc_language
-                    )
+                    summary, _ = self.generate_summary(tenant_id, preview.content, summary_index_setting)
                     preview.summary = summary
             else:
                 # Fallback: try without app context (may fail)
-                summary, _ = self.generate_summary(
-                    tenant_id, preview.content, summary_index_setting, document_language=doc_language
-                )
+                summary, _ = self.generate_summary(tenant_id, preview.content, summary_index_setting)
                 preview.summary = summary
 
         # Generate summaries concurrently using ThreadPoolExecutor
@@ -364,7 +356,6 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
         text: str,
         summary_index_setting: dict | None = None,
         segment_id: str | None = None,
-        document_language: str | None = None,
     ) -> tuple[str, LLMUsage]:
         """
         Generate summary for the given text using ModelInstance.invoke_llm and the default or custom summary prompt,
@@ -375,8 +366,6 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
             text: Text content to summarize
             summary_index_setting: Summary index configuration
             segment_id: Optional segment ID to fetch attachments from SegmentAttachmentBinding table
-            document_language: Optional document language (e.g., "Chinese", "English")
-                to ensure summary is generated in the correct language
 
         Returns:
             Tuple of (summary_content, llm_usage) where llm_usage is LLMUsage object
@@ -392,22 +381,8 @@ class ParagraphIndexProcessor(BaseIndexProcessor):
             raise ValueError("model_name and model_provider_name are required in summary_index_setting")
 
         # Import default summary prompt
-        is_default_prompt = False
         if not summary_prompt:
             summary_prompt = DEFAULT_GENERATOR_SUMMARY_PROMPT
-            is_default_prompt = True
-
-        # Format prompt with document language only for default prompt
-        # Custom prompts are used as-is to avoid interfering with user-defined templates
-        # If document_language is provided, use it; otherwise, use "the same language as the input content"
-        # This is especially important for image-only chunks where text is empty or minimal
-        if is_default_prompt:
-            language_for_prompt = document_language or "the same language as the input content"
-            try:
-                summary_prompt = summary_prompt.format(language=language_for_prompt)
-            except KeyError:
-                # If default prompt doesn't have {language} placeholder, use it as-is
-                pass
 
         provider_manager = ProviderManager()
         provider_model_bundle = provider_manager.get_provider_model_bundle(

api/core/rag/index_processor/processor/parent_child_index_processor.py

@@ -358,11 +358,7 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
         }
 
     def generate_summary_preview(
-        self,
-        tenant_id: str,
-        preview_texts: list[PreviewDetail],
-        summary_index_setting: dict,
-        doc_language: str | None = None,
+        self, tenant_id: str, preview_texts: list[PreviewDetail], summary_index_setting: dict
     ) -> list[PreviewDetail]:
         """
         For each parent chunk in preview_texts, concurrently call generate_summary to generate a summary
@@ -393,7 +389,6 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
                         tenant_id=tenant_id,
                         text=preview.content,
                         summary_index_setting=summary_index_setting,
-                        document_language=doc_language,
                     )
                     preview.summary = summary
             else:
@@ -402,7 +397,6 @@ class ParentChildIndexProcessor(BaseIndexProcessor):
                     tenant_id=tenant_id,
                     text=preview.content,
                     summary_index_setting=summary_index_setting,
-                    document_language=doc_language,
                 )
                 preview.summary = summary
 

api/core/rag/index_processor/processor/qa_index_processor.py

@@ -241,11 +241,7 @@ class QAIndexProcessor(BaseIndexProcessor):
         }
 
     def generate_summary_preview(
-        self,
-        tenant_id: str,
-        preview_texts: list[PreviewDetail],
-        summary_index_setting: dict,
-        doc_language: str | None = None,
+        self, tenant_id: str, preview_texts: list[PreviewDetail], summary_index_setting: dict
     ) -> list[PreviewDetail]:
         """
         QA model doesn't generate summaries, so this method returns preview_texts unchanged.

api/core/rag/retrieval/dataset_retrieval.py

@@ -27,7 +27,8 @@ from core.model_runtime.entities.llm_entities import LLMResult, LLMUsage
 from core.model_runtime.entities.message_entities import PromptMessage, PromptMessageRole, PromptMessageTool
 from core.model_runtime.entities.model_entities import ModelFeature, ModelType
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
-from core.ops.ops_trace_manager import TraceQueueManager
+from core.ops.entities.trace_entity import TraceTaskName
+from core.ops.ops_trace_manager import TraceQueueManager, TraceTask
 from core.ops.utils import measure_time
 from core.prompt.advanced_prompt_transform import AdvancedPromptTransform
 from core.prompt.entities.advanced_prompt_entities import ChatModelMessage, CompletionModelPromptTemplate
@@ -55,8 +56,6 @@ from core.rag.retrieval.template_prompts import (
     METADATA_FILTER_USER_PROMPT_2,
     METADATA_FILTER_USER_PROMPT_3,
 )
-from core.telemetry import TelemetryContext, TelemetryEvent, TraceTaskName
-from core.telemetry import emit as telemetry_emit
 from core.tools.signature import sign_upload_file
 from core.tools.utils.dataset_retriever.dataset_retriever_base_tool import DatasetRetrieverBaseTool
 from extensions.ext_database import db
@@ -729,21 +728,10 @@ class DatasetRetrieval:
             self.application_generate_entity.trace_manager if self.application_generate_entity else None
         )
         if trace_manager:
-            app_config = self.application_generate_entity.app_config if self.application_generate_entity else None
-            telemetry_emit(
-                TelemetryEvent(
-                    name=TraceTaskName.DATASET_RETRIEVAL_TRACE,
-                    context=TelemetryContext(
-                        tenant_id=app_config.tenant_id if app_config else None,
-                        app_id=app_config.app_id if app_config else None,
-                    ),
-                    payload={
-                        "message_id": message_id,
-                        "documents": documents,
-                        "timer": timer,
-                    },
-                ),
-                trace_manager=trace_manager,
+            trace_manager.add_trace_task(
+                TraceTask(
+                    TraceTaskName.DATASET_RETRIEVAL_TRACE, message_id=message_id, documents=documents, timer=timer
+                )
             )
 
     def _on_query(

api/core/telemetry/__init__.py

@@ -1,43 +0,0 @@
-"""Telemetry facade.
-
-Thin public API for emitting telemetry events.  All routing logic
-lives in ``core.telemetry.gateway`` which is shared by both CE and EE.
-"""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-from core.ops.entities.trace_entity import TraceTaskName
-from core.telemetry.events import TelemetryContext, TelemetryEvent
-from core.telemetry.gateway import emit as gateway_emit
-from core.telemetry.gateway import get_trace_task_to_case
-
-if TYPE_CHECKING:
-    from core.ops.ops_trace_manager import TraceQueueManager
-
-
-def emit(event: TelemetryEvent, trace_manager: TraceQueueManager | None = None) -> None:
-    """Emit a telemetry event.
-
-    Translates the ``TelemetryEvent`` (keyed by ``TraceTaskName``) into a
-    ``TelemetryCase`` and delegates to ``core.telemetry.gateway.emit()``.
-    """
-    case = get_trace_task_to_case().get(event.name)
-    if case is None:
-        return
-
-    context: dict[str, object] = {
-        "tenant_id": event.context.tenant_id,
-        "user_id": event.context.user_id,
-        "app_id": event.context.app_id,
-    }
-    gateway_emit(case, context, event.payload, trace_manager)
-
-
-__all__ = [
-    "TelemetryContext",
-    "TelemetryEvent",
-    "TraceTaskName",
-    "emit",
-]

api/core/telemetry/events.py

@@ -1,21 +0,0 @@
-from __future__ import annotations
-
-from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
-    from core.ops.entities.trace_entity import TraceTaskName
-
-
-@dataclass(frozen=True)
-class TelemetryContext:
-    tenant_id: str | None = None
-    user_id: str | None = None
-    app_id: str | None = None
-
-
-@dataclass(frozen=True)
-class TelemetryEvent:
-    name: TraceTaskName
-    context: TelemetryContext
-    payload: dict[str, Any]

api/core/telemetry/gateway.py

@@ -1,233 +0,0 @@
-"""Telemetry gateway — single routing layer for all editions.
-
-Maps ``TelemetryCase`` → ``CaseRoute`` and dispatches events to either
-the CE/EE trace pipeline (``TraceQueueManager``) or the enterprise-only
-metric/log Celery queue.
-
-This module lives in ``core/`` so both CE and EE share one routing table
-and one ``emit()`` entry point.  No separate enterprise gateway module is
-needed — enterprise-specific dispatch (Celery task, payload offloading)
-is handled here behind lazy imports that no-op in CE.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import uuid
-from typing import TYPE_CHECKING, Any
-
-from core.ops.entities.trace_entity import TraceTaskName
-from extensions.ext_storage import storage
-
-if TYPE_CHECKING:
-    from core.ops.ops_trace_manager import TraceQueueManager
-    from enterprise.telemetry.contracts import TelemetryCase
-
-logger = logging.getLogger(__name__)
-
-PAYLOAD_SIZE_THRESHOLD_BYTES = 1 * 1024 * 1024
-
-# ---------------------------------------------------------------------------
-# Routing table — authoritative mapping for all editions
-# ---------------------------------------------------------------------------
-
-_case_to_trace_task: dict | None = None
-_case_routing: dict | None = None
-
-
-def _get_case_to_trace_task() -> dict:
-    global _case_to_trace_task
-    if _case_to_trace_task is None:
-        from enterprise.telemetry.contracts import TelemetryCase
-
-        _case_to_trace_task = {
-            TelemetryCase.WORKFLOW_RUN: TraceTaskName.WORKFLOW_TRACE,
-            TelemetryCase.MESSAGE_RUN: TraceTaskName.MESSAGE_TRACE,
-            TelemetryCase.NODE_EXECUTION: TraceTaskName.NODE_EXECUTION_TRACE,
-            TelemetryCase.DRAFT_NODE_EXECUTION: TraceTaskName.DRAFT_NODE_EXECUTION_TRACE,
-            TelemetryCase.PROMPT_GENERATION: TraceTaskName.PROMPT_GENERATION_TRACE,
-            TelemetryCase.TOOL_EXECUTION: TraceTaskName.TOOL_TRACE,
-            TelemetryCase.MODERATION_CHECK: TraceTaskName.MODERATION_TRACE,
-            TelemetryCase.SUGGESTED_QUESTION: TraceTaskName.SUGGESTED_QUESTION_TRACE,
-            TelemetryCase.DATASET_RETRIEVAL: TraceTaskName.DATASET_RETRIEVAL_TRACE,
-            TelemetryCase.GENERATE_NAME: TraceTaskName.GENERATE_NAME_TRACE,
-        }
-    return _case_to_trace_task
-
-
-def get_trace_task_to_case() -> dict:
-    """Return TraceTaskName → TelemetryCase (inverse of _get_case_to_trace_task)."""
-    return {v: k for k, v in _get_case_to_trace_task().items()}
-
-
-def _get_case_routing() -> dict:
-    global _case_routing
-    if _case_routing is None:
-        from enterprise.telemetry.contracts import CaseRoute, SignalType, TelemetryCase
-
-        _case_routing = {
-            # TRACE — CE-eligible (flow in both CE and EE)
-            TelemetryCase.WORKFLOW_RUN: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=True),
-            TelemetryCase.MESSAGE_RUN: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=True),
-            TelemetryCase.TOOL_EXECUTION: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=True),
-            TelemetryCase.MODERATION_CHECK: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=True),
-            TelemetryCase.SUGGESTED_QUESTION: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=True),
-            TelemetryCase.DATASET_RETRIEVAL: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=True),
-            TelemetryCase.GENERATE_NAME: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=True),
-            # TRACE — enterprise-only
-            TelemetryCase.NODE_EXECUTION: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=False),
-            TelemetryCase.DRAFT_NODE_EXECUTION: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=False),
-            TelemetryCase.PROMPT_GENERATION: CaseRoute(signal_type=SignalType.TRACE, ce_eligible=False),
-            # METRIC_LOG — enterprise-only (signal-driven, not trace)
-            TelemetryCase.APP_CREATED: CaseRoute(signal_type=SignalType.METRIC_LOG, ce_eligible=False),
-            TelemetryCase.APP_UPDATED: CaseRoute(signal_type=SignalType.METRIC_LOG, ce_eligible=False),
-            TelemetryCase.APP_DELETED: CaseRoute(signal_type=SignalType.METRIC_LOG, ce_eligible=False),
-            TelemetryCase.FEEDBACK_CREATED: CaseRoute(signal_type=SignalType.METRIC_LOG, ce_eligible=False),
-        }
-    return _case_routing
-
-# Public exports for tests and external consumers
-CASE_TO_TRACE_TASK = _get_case_to_trace_task()
-CASE_ROUTING = _get_case_routing()
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def is_enterprise_telemetry_enabled() -> bool:
-    try:
-        from enterprise.telemetry.exporter import is_enterprise_telemetry_enabled
-
-        return is_enterprise_telemetry_enabled()
-    except Exception:
-        return False
-
-
-def _handle_payload_sizing(
-    payload: dict[str, Any],
-    tenant_id: str,
-    event_id: str,
-) -> tuple[dict[str, Any], str | None]:
-    """Inline or offload payload based on size.
-
-    Returns ``(payload_for_envelope, storage_key | None)``.  Payloads
-    exceeding ``PAYLOAD_SIZE_THRESHOLD_BYTES`` are written to object
-    storage and replaced with an empty dict in the envelope.
-    """
-    try:
-        payload_json = json.dumps(payload)
-        payload_size = len(payload_json.encode("utf-8"))
-    except (TypeError, ValueError):
-        logger.warning("Failed to serialize payload for sizing: event_id=%s", event_id)
-        return payload, None
-
-    if payload_size <= PAYLOAD_SIZE_THRESHOLD_BYTES:
-        return payload, None
-
-    storage_key = f"telemetry/{tenant_id}/{event_id}.json"
-    try:
-        storage.save(storage_key, payload_json.encode("utf-8"))
-        logger.debug("Stored large payload to storage: key=%s, size=%d", storage_key, payload_size)
-        return {}, storage_key
-    except Exception:
-        logger.warning("Failed to store large payload, inlining instead: event_id=%s", event_id, exc_info=True)
-        return payload, None
-
-
-# ---------------------------------------------------------------------------
-# Public API
-# ---------------------------------------------------------------------------
-
-
-def emit(
-    case: TelemetryCase,
-    context: dict[str, Any],
-    payload: dict[str, Any],
-    trace_manager: TraceQueueManager | None = None,
-) -> None:
-    """Route a telemetry event to the correct pipeline.
-
-    TRACE events are enqueued into ``TraceQueueManager`` (works in both CE
-    and EE).  Enterprise-only traces are silently dropped when EE is
-    disabled.
-
-    METRIC_LOG events are dispatched to the enterprise Celery queue;
-    silently dropped when enterprise telemetry is unavailable.
-    """
-    route = _get_case_routing().get(case)
-    if route is None:
-        logger.warning("Unknown telemetry case: %s, dropping event", case)
-        return
-
-    if not route.ce_eligible and not is_enterprise_telemetry_enabled():
-        logger.debug("Dropping EE-only event: case=%s (EE disabled)", case)
-        return
-
-    if route.signal_type == "trace":
-        _emit_trace(case, context, payload, trace_manager)
-    else:
-        _emit_metric_log(case, context, payload)
-
-
-def _emit_trace(
-    case: TelemetryCase,
-    context: dict[str, Any],
-    payload: dict[str, Any],
-    trace_manager: TraceQueueManager | None,
-) -> None:
-    from core.ops.ops_trace_manager import TraceQueueManager as LocalTraceQueueManager
-    from core.ops.ops_trace_manager import TraceTask
-
-    trace_task_name = _get_case_to_trace_task().get(case)
-    if trace_task_name is None:
-        logger.warning("No TraceTaskName mapping for case: %s", case)
-        return
-
-    queue_manager = trace_manager or LocalTraceQueueManager(
-        app_id=context.get("app_id"),
-        user_id=context.get("user_id"),
-    )
-    queue_manager.add_trace_task(TraceTask(trace_task_name, **payload))
-    logger.debug("Enqueued trace task: case=%s, app_id=%s", case, context.get("app_id"))
-
-
-def _emit_metric_log(
-    case: TelemetryCase,
-    context: dict[str, Any],
-    payload: dict[str, Any],
-) -> None:
-    """Build envelope and dispatch to enterprise Celery queue.
-
-    No-ops when the enterprise telemetry task is not importable (CE mode).
-    """
-    try:
-        from tasks.enterprise_telemetry_task import process_enterprise_telemetry
-    except ImportError:
-        logger.debug("Enterprise metric/log dispatch unavailable, dropping: case=%s", case)
-        return
-
-    tenant_id = context.get("tenant_id", "")
-    event_id = str(uuid.uuid4())
-
-    payload_for_envelope, payload_ref = _handle_payload_sizing(payload, tenant_id, event_id)
-
-    from enterprise.telemetry.contracts import TelemetryEnvelope
-
-    envelope = TelemetryEnvelope(
-        case=case,
-        tenant_id=tenant_id,
-        event_id=event_id,
-        payload=payload_for_envelope,
-        metadata={"payload_ref": payload_ref} if payload_ref else None,
-    )
-
-    process_enterprise_telemetry.delay(envelope.model_dump_json())
-    logger.debug(
-        "Enqueued metric/log event: case=%s, tenant_id=%s, event_id=%s",
-        case,
-        tenant_id,
-        event_id,
-    )

api/core/tools/workflow_as_tool/tool.py

@@ -50,7 +50,6 @@ class WorkflowTool(Tool):
         self.workflow_call_depth = workflow_call_depth
         self.label = label
         self._latest_usage = LLMUsage.empty_usage()
-        self.parent_trace_context: dict[str, str] | None = None
 
         super().__init__(entity=entity, runtime=runtime)
 
@@ -91,15 +90,11 @@ class WorkflowTool(Tool):
 
         self._latest_usage = LLMUsage.empty_usage()
 
-        args: dict[str, Any] = {"inputs": tool_parameters, "files": files}
-        if self.parent_trace_context:
-            args["_parent_trace_context"] = self.parent_trace_context
-
         result = generator.generate(
             app_model=app,
             workflow=workflow,
             user=user,
-            args=args,
+            args={"inputs": tool_parameters, "files": files},
             invoke_from=self.runtime.invoke_from,
             streaming=False,
             call_depth=self.workflow_call_depth + 1,

api/core/workflow/enums.py

@@ -232,8 +232,6 @@ class WorkflowNodeExecutionMetadataKey(StrEnum):
     """
 
     TOTAL_TOKENS = "total_tokens"
-    PROMPT_TOKENS = "prompt_tokens"
-    COMPLETION_TOKENS = "completion_tokens"
     TOTAL_PRICE = "total_price"
     CURRENCY = "currency"
     TOOL_INFO = "tool_info"

api/core/workflow/nodes/agent/agent_node.py

@@ -192,33 +192,32 @@ class AgentNode(Node[AgentNodeData]):
                 result[parameter_name] = None
                 continue
             agent_input = node_data.agent_parameters[parameter_name]
-            match agent_input.type:
-                case "variable":
-                    variable = variable_pool.get(agent_input.value)  # type: ignore
-                    if variable is None:
-                        raise AgentVariableNotFoundError(str(agent_input.value))
-                    parameter_value = variable.value
-                case "mixed" | "constant":
-                    # variable_pool.convert_template expects a string template,
-                    # but if passing a dict, convert to JSON string first before rendering
-                    try:
-                        if not isinstance(agent_input.value, str):
-                            parameter_value = json.dumps(agent_input.value, ensure_ascii=False)
-                        else:
-                            parameter_value = str(agent_input.value)
-                    except TypeError:
+            if agent_input.type == "variable":
+                variable = variable_pool.get(agent_input.value)  # type: ignore
+                if variable is None:
+                    raise AgentVariableNotFoundError(str(agent_input.value))
+                parameter_value = variable.value
+            elif agent_input.type in {"mixed", "constant"}:
+                # variable_pool.convert_template expects a string template,
+                # but if passing a dict, convert to JSON string first before rendering
+                try:
+                    if not isinstance(agent_input.value, str):
+                        parameter_value = json.dumps(agent_input.value, ensure_ascii=False)
+                    else:
                         parameter_value = str(agent_input.value)
-                    segment_group = variable_pool.convert_template(parameter_value)
-                    parameter_value = segment_group.log if for_log else segment_group.text
-                    # variable_pool.convert_template returns a string,
-                    # so we need to convert it back to a dictionary
-                    try:
-                        if not isinstance(agent_input.value, str):
-                            parameter_value = json.loads(parameter_value)
-                    except json.JSONDecodeError:
-                        parameter_value = parameter_value
-                case _:
-                    raise AgentInputTypeError(agent_input.type)
+                except TypeError:
+                    parameter_value = str(agent_input.value)
+                segment_group = variable_pool.convert_template(parameter_value)
+                parameter_value = segment_group.log if for_log else segment_group.text
+                # variable_pool.convert_template returns a string,
+                # so we need to convert it back to a dictionary
+                try:
+                    if not isinstance(agent_input.value, str):
+                        parameter_value = json.loads(parameter_value)
+                except json.JSONDecodeError:
+                    parameter_value = parameter_value
+            else:
+                raise AgentInputTypeError(agent_input.type)
             value = parameter_value
             if parameter.type == "array[tools]":
                 value = cast(list[dict[str, Any]], value)
@@ -375,13 +374,12 @@ class AgentNode(Node[AgentNodeData]):
         result: dict[str, Any] = {}
         for parameter_name in typed_node_data.agent_parameters:
             input = typed_node_data.agent_parameters[parameter_name]
-            match input.type:
-                case "mixed" | "constant":
-                    selectors = VariableTemplateParser(str(input.value)).extract_variable_selectors()
-                    for selector in selectors:
-                        result[selector.variable] = selector.value_selector
-                case "variable":
-                    result[parameter_name] = input.value
+            if input.type in ["mixed", "constant"]:
+                selectors = VariableTemplateParser(str(input.value)).extract_variable_selectors()
+                for selector in selectors:
+                    result[selector.variable] = selector.value_selector
+            elif input.type == "variable":
+                result[parameter_name] = input.value
 
         result = {node_id + "." + key: value for key, value in result.items()}
 

api/core/workflow/nodes/datasource/datasource_node.py

@@ -270,18 +270,15 @@ class DatasourceNode(Node[DatasourceNodeData]):
         if typed_node_data.datasource_parameters:
             for parameter_name in typed_node_data.datasource_parameters:
                 input = typed_node_data.datasource_parameters[parameter_name]
-                match input.type:
-                    case "mixed":
-                        assert isinstance(input.value, str)
-                        selectors = VariableTemplateParser(input.value).extract_variable_selectors()
-                        for selector in selectors:
-                            result[selector.variable] = selector.value_selector
-                    case "variable":
-                        result[parameter_name] = input.value
-                    case "constant":
-                        pass
-                    case None:
-                        pass
+                if input.type == "mixed":
+                    assert isinstance(input.value, str)
+                    selectors = VariableTemplateParser(input.value).extract_variable_selectors()
+                    for selector in selectors:
+                        result[selector.variable] = selector.value_selector
+                elif input.type == "variable":
+                    result[parameter_name] = input.value
+                elif input.type == "constant":
+                    pass
 
             result = {node_id + "." + key: value for key, value in result.items()}
 
@@ -311,107 +308,99 @@ class DatasourceNode(Node[DatasourceNodeData]):
         variables: dict[str, Any] = {}
 
         for message in message_stream:
-            match message.type:
-                case (
-                    DatasourceMessage.MessageType.IMAGE_LINK
-                    | DatasourceMessage.MessageType.BINARY_LINK
-                    | DatasourceMessage.MessageType.IMAGE
-                ):
-                    assert isinstance(message.message, DatasourceMessage.TextMessage)
+            if message.type in {
+                DatasourceMessage.MessageType.IMAGE_LINK,
+                DatasourceMessage.MessageType.BINARY_LINK,
+                DatasourceMessage.MessageType.IMAGE,
+            }:
+                assert isinstance(message.message, DatasourceMessage.TextMessage)
 
-                    url = message.message.text
-                    transfer_method = FileTransferMethod.TOOL_FILE
+                url = message.message.text
+                transfer_method = FileTransferMethod.TOOL_FILE
 
-                    datasource_file_id = str(url).split("/")[-1].split(".")[0]
+                datasource_file_id = str(url).split("/")[-1].split(".")[0]
 
-                    with Session(db.engine) as session:
-                        stmt = select(ToolFile).where(ToolFile.id == datasource_file_id)
-                        datasource_file = session.scalar(stmt)
-                        if datasource_file is None:
-                            raise ToolFileError(f"Tool file {datasource_file_id} does not exist")
+                with Session(db.engine) as session:
+                    stmt = select(ToolFile).where(ToolFile.id == datasource_file_id)
+                    datasource_file = session.scalar(stmt)
+                    if datasource_file is None:
+                        raise ToolFileError(f"Tool file {datasource_file_id} does not exist")
 
-                    mapping = {
-                        "tool_file_id": datasource_file_id,
-                        "type": file_factory.get_file_type_by_mime_type(datasource_file.mimetype),
-                        "transfer_method": transfer_method,
-                        "url": url,
-                    }
-                    file = file_factory.build_from_mapping(
+                mapping = {
+                    "tool_file_id": datasource_file_id,
+                    "type": file_factory.get_file_type_by_mime_type(datasource_file.mimetype),
+                    "transfer_method": transfer_method,
+                    "url": url,
+                }
+                file = file_factory.build_from_mapping(
+                    mapping=mapping,
+                    tenant_id=self.tenant_id,
+                )
+                files.append(file)
+            elif message.type == DatasourceMessage.MessageType.BLOB:
+                # get tool file id
+                assert isinstance(message.message, DatasourceMessage.TextMessage)
+                assert message.meta
+
+                datasource_file_id = message.message.text.split("/")[-1].split(".")[0]
+                with Session(db.engine) as session:
+                    stmt = select(ToolFile).where(ToolFile.id == datasource_file_id)
+                    datasource_file = session.scalar(stmt)
+                    if datasource_file is None:
+                        raise ToolFileError(f"datasource file {datasource_file_id} not exists")
+
+                mapping = {
+                    "tool_file_id": datasource_file_id,
+                    "transfer_method": FileTransferMethod.TOOL_FILE,
+                }
+
+                files.append(
+                    file_factory.build_from_mapping(
                         mapping=mapping,
                         tenant_id=self.tenant_id,
                     )
-                    files.append(file)
-                case DatasourceMessage.MessageType.BLOB:
-                    # get tool file id
-                    assert isinstance(message.message, DatasourceMessage.TextMessage)
-                    assert message.meta
+                )
+            elif message.type == DatasourceMessage.MessageType.TEXT:
+                assert isinstance(message.message, DatasourceMessage.TextMessage)
+                text += message.message.text
+                yield StreamChunkEvent(
+                    selector=[self._node_id, "text"],
+                    chunk=message.message.text,
+                    is_final=False,
+                )
+            elif message.type == DatasourceMessage.MessageType.JSON:
+                assert isinstance(message.message, DatasourceMessage.JsonMessage)
+                json.append(message.message.json_object)
+            elif message.type == DatasourceMessage.MessageType.LINK:
+                assert isinstance(message.message, DatasourceMessage.TextMessage)
+                stream_text = f"Link: {message.message.text}\n"
+                text += stream_text
+                yield StreamChunkEvent(
+                    selector=[self._node_id, "text"],
+                    chunk=stream_text,
+                    is_final=False,
+                )
+            elif message.type == DatasourceMessage.MessageType.VARIABLE:
+                assert isinstance(message.message, DatasourceMessage.VariableMessage)
+                variable_name = message.message.variable_name
+                variable_value = message.message.variable_value
+                if message.message.stream:
+                    if not isinstance(variable_value, str):
+                        raise ValueError("When 'stream' is True, 'variable_value' must be a string.")
+                    if variable_name not in variables:
+                        variables[variable_name] = ""
+                    variables[variable_name] += variable_value
 
-                    datasource_file_id = message.message.text.split("/")[-1].split(".")[0]
-                    with Session(db.engine) as session:
-                        stmt = select(ToolFile).where(ToolFile.id == datasource_file_id)
-                        datasource_file = session.scalar(stmt)
-                        if datasource_file is None:
-                            raise ToolFileError(f"datasource file {datasource_file_id} not exists")
-
-                    mapping = {
-                        "tool_file_id": datasource_file_id,
-                        "transfer_method": FileTransferMethod.TOOL_FILE,
-                    }
-
-                    files.append(
-                        file_factory.build_from_mapping(
-                            mapping=mapping,
-                            tenant_id=self.tenant_id,
-                        )
-                    )
-                case DatasourceMessage.MessageType.TEXT:
-                    assert isinstance(message.message, DatasourceMessage.TextMessage)
-                    text += message.message.text
                     yield StreamChunkEvent(
-                        selector=[self._node_id, "text"],
-                        chunk=message.message.text,
+                        selector=[self._node_id, variable_name],
+                        chunk=variable_value,
                         is_final=False,
                     )
-                case DatasourceMessage.MessageType.JSON:
-                    assert isinstance(message.message, DatasourceMessage.JsonMessage)
-                    json.append(message.message.json_object)
-                case DatasourceMessage.MessageType.LINK:
-                    assert isinstance(message.message, DatasourceMessage.TextMessage)
-                    stream_text = f"Link: {message.message.text}\n"
-                    text += stream_text
-                    yield StreamChunkEvent(
-                        selector=[self._node_id, "text"],
-                        chunk=stream_text,
-                        is_final=False,
-                    )
-                case DatasourceMessage.MessageType.VARIABLE:
-                    assert isinstance(message.message, DatasourceMessage.VariableMessage)
-                    variable_name = message.message.variable_name
-                    variable_value = message.message.variable_value
-                    if message.message.stream:
-                        if not isinstance(variable_value, str):
-                            raise ValueError("When 'stream' is True, 'variable_value' must be a string.")
-                        if variable_name not in variables:
-                            variables[variable_name] = ""
-                        variables[variable_name] += variable_value
-
-                        yield StreamChunkEvent(
-                            selector=[self._node_id, variable_name],
-                            chunk=variable_value,
-                            is_final=False,
-                        )
-                    else:
-                        variables[variable_name] = variable_value
-                case DatasourceMessage.MessageType.FILE:
-                    assert message.meta is not None
-                    files.append(message.meta["file"])
-                case (
-                    DatasourceMessage.MessageType.BLOB_CHUNK
-                    | DatasourceMessage.MessageType.LOG
-                    | DatasourceMessage.MessageType.RETRIEVER_RESOURCES
-                ):
-                    pass
-
+                else:
+                    variables[variable_name] = variable_value
+            elif message.type == DatasourceMessage.MessageType.FILE:
+                assert message.meta is not None
+                files.append(message.meta["file"])
         # mark the end of the stream
         yield StreamChunkEvent(
             selector=[self._node_id, "text"],

api/core/workflow/nodes/knowledge_index/knowledge_index_node.py

@@ -78,21 +78,12 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
                 indexing_technique = node_data.indexing_technique or dataset.indexing_technique
                 summary_index_setting = node_data.summary_index_setting or dataset.summary_index_setting
 
-                # Try to get document language if document_id is available
-                doc_language = None
-                document_id = variable_pool.get(["sys", SystemVariableKey.DOCUMENT_ID])
-                if document_id:
-                    document = db.session.query(Document).filter_by(id=document_id.value).first()
-                    if document and document.doc_language:
-                        doc_language = document.doc_language
-
                 outputs = self._get_preview_output_with_summaries(
                     node_data.chunk_structure,
                     chunks,
                     dataset=dataset,
                     indexing_technique=indexing_technique,
                     summary_index_setting=summary_index_setting,
-                    doc_language=doc_language,
                 )
                 return NodeRunResult(
                     status=WorkflowNodeExecutionStatus.SUCCEEDED,
@@ -324,7 +315,6 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
         dataset: Dataset,
         indexing_technique: str | None = None,
         summary_index_setting: dict | None = None,
-        doc_language: str | None = None,
     ) -> Mapping[str, Any]:
         """
         Generate preview output with summaries for chunks in preview mode.
@@ -336,7 +326,6 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
             dataset: Dataset object (for tenant_id)
             indexing_technique: Indexing technique from node config or dataset
             summary_index_setting: Summary index setting from node config or dataset
-            doc_language: Optional document language to ensure summary is generated in the correct language
         """
         index_processor = IndexProcessorFactory(chunk_structure).init_index_processor()
         preview_output = index_processor.format_preview(chunks)
@@ -376,7 +365,6 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
                                 tenant_id=dataset.tenant_id,
                                 text=preview_item["content"],
                                 summary_index_setting=summary_index_setting,
-                                document_language=doc_language,
                             )
                             if summary:
                                 preview_item["summary"] = summary
@@ -386,7 +374,6 @@ class KnowledgeIndexNode(Node[KnowledgeIndexNodeData]):
                             tenant_id=dataset.tenant_id,
                             text=preview_item["content"],
                             summary_index_setting=summary_index_setting,
-                            document_language=doc_language,
                         )
                         if summary:
                             preview_item["summary"] = summary

api/core/workflow/nodes/knowledge_retrieval/knowledge_retrieval_node.py

@@ -303,34 +303,33 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
         elif str(node_data.retrieval_mode) == DatasetRetrieveConfigEntity.RetrieveStrategy.MULTIPLE:
             if node_data.multiple_retrieval_config is None:
                 raise ValueError("multiple_retrieval_config is required")
-            match node_data.multiple_retrieval_config.reranking_mode:
-                case "reranking_model":
-                    if node_data.multiple_retrieval_config.reranking_model:
-                        reranking_model = {
-                            "reranking_provider_name": node_data.multiple_retrieval_config.reranking_model.provider,
-                            "reranking_model_name": node_data.multiple_retrieval_config.reranking_model.model,
-                        }
-                    else:
-                        reranking_model = None
-                    weights = None
-                case "weighted_score":
-                    if node_data.multiple_retrieval_config.weights is None:
-                        raise ValueError("weights is required")
-                    reranking_model = None
-                    vector_setting = node_data.multiple_retrieval_config.weights.vector_setting
-                    weights = {
-                        "vector_setting": {
-                            "vector_weight": vector_setting.vector_weight,
-                            "embedding_provider_name": vector_setting.embedding_provider_name,
-                            "embedding_model_name": vector_setting.embedding_model_name,
-                        },
-                        "keyword_setting": {
-                            "keyword_weight": node_data.multiple_retrieval_config.weights.keyword_setting.keyword_weight
-                        },
+            if node_data.multiple_retrieval_config.reranking_mode == "reranking_model":
+                if node_data.multiple_retrieval_config.reranking_model:
+                    reranking_model = {
+                        "reranking_provider_name": node_data.multiple_retrieval_config.reranking_model.provider,
+                        "reranking_model_name": node_data.multiple_retrieval_config.reranking_model.model,
                     }
-                case _:
+                else:
                     reranking_model = None
-                    weights = None
+                weights = None
+            elif node_data.multiple_retrieval_config.reranking_mode == "weighted_score":
+                if node_data.multiple_retrieval_config.weights is None:
+                    raise ValueError("weights is required")
+                reranking_model = None
+                vector_setting = node_data.multiple_retrieval_config.weights.vector_setting
+                weights = {
+                    "vector_setting": {
+                        "vector_weight": vector_setting.vector_weight,
+                        "embedding_provider_name": vector_setting.embedding_provider_name,
+                        "embedding_model_name": vector_setting.embedding_model_name,
+                    },
+                    "keyword_setting": {
+                        "keyword_weight": node_data.multiple_retrieval_config.weights.keyword_setting.keyword_weight
+                    },
+                }
+            else:
+                reranking_model = None
+                weights = None
             all_documents = dataset_retrieval.multiple_retrieve(
                 app_id=self.app_id,
                 tenant_id=self.tenant_id,
@@ -454,74 +453,73 @@ class KnowledgeRetrievalNode(LLMUsageTrackingMixin, Node[KnowledgeRetrievalNodeD
         )
         filters: list[Any] = []
         metadata_condition = None
-        match node_data.metadata_filtering_mode:
-            case "disabled":
-                return None, None, usage
-            case "automatic":
-                automatic_metadata_filters, automatic_usage = self._automatic_metadata_filter_func(
-                    dataset_ids, query, node_data
+        if node_data.metadata_filtering_mode == "disabled":
+            return None, None, usage
+        elif node_data.metadata_filtering_mode == "automatic":
+            automatic_metadata_filters, automatic_usage = self._automatic_metadata_filter_func(
+                dataset_ids, query, node_data
+            )
+            usage = self._merge_usage(usage, automatic_usage)
+            if automatic_metadata_filters:
+                conditions = []
+                for sequence, filter in enumerate(automatic_metadata_filters):
+                    DatasetRetrieval.process_metadata_filter_func(
+                        sequence,
+                        filter.get("condition", ""),
+                        filter.get("metadata_name", ""),
+                        filter.get("value"),
+                        filters,
+                    )
+                    conditions.append(
+                        Condition(
+                            name=filter.get("metadata_name"),  # type: ignore
+                            comparison_operator=filter.get("condition"),  # type: ignore
+                            value=filter.get("value"),
+                        )
+                    )
+                metadata_condition = MetadataCondition(
+                    logical_operator=node_data.metadata_filtering_conditions.logical_operator
+                    if node_data.metadata_filtering_conditions
+                    else "or",
+                    conditions=conditions,
                 )
-                usage = self._merge_usage(usage, automatic_usage)
-                if automatic_metadata_filters:
-                    conditions = []
-                    for sequence, filter in enumerate(automatic_metadata_filters):
-                        DatasetRetrieval.process_metadata_filter_func(
-                            sequence,
-                            filter.get("condition", ""),
-                            filter.get("metadata_name", ""),
-                            filter.get("value"),
-                            filters,
+        elif node_data.metadata_filtering_mode == "manual":
+            if node_data.metadata_filtering_conditions:
+                conditions = []
+                for sequence, condition in enumerate(node_data.metadata_filtering_conditions.conditions):  # type: ignore
+                    metadata_name = condition.name
+                    expected_value = condition.value
+                    if expected_value is not None and condition.comparison_operator not in ("empty", "not empty"):
+                        if isinstance(expected_value, str):
+                            expected_value = self.graph_runtime_state.variable_pool.convert_template(
+                                expected_value
+                            ).value[0]
+                            if expected_value.value_type in {"number", "integer", "float"}:
+                                expected_value = expected_value.value
+                            elif expected_value.value_type == "string":
+                                expected_value = re.sub(r"[\r\n\t]+", " ", expected_value.text).strip()
+                            else:
+                                raise ValueError("Invalid expected metadata value type")
+                    conditions.append(
+                        Condition(
+                            name=metadata_name,
+                            comparison_operator=condition.comparison_operator,
+                            value=expected_value,
                         )
-                        conditions.append(
-                            Condition(
-                                name=filter.get("metadata_name"),  # type: ignore
-                                comparison_operator=filter.get("condition"),  # type: ignore
-                                value=filter.get("value"),
-                            )
-                        )
-                    metadata_condition = MetadataCondition(
-                        logical_operator=node_data.metadata_filtering_conditions.logical_operator
-                        if node_data.metadata_filtering_conditions
-                        else "or",
-                        conditions=conditions,
                     )
-            case "manual":
-                if node_data.metadata_filtering_conditions:
-                    conditions = []
-                    for sequence, condition in enumerate(node_data.metadata_filtering_conditions.conditions):  # type: ignore
-                        metadata_name = condition.name
-                        expected_value = condition.value
-                        if expected_value is not None and condition.comparison_operator not in ("empty", "not empty"):
-                            if isinstance(expected_value, str):
-                                expected_value = self.graph_runtime_state.variable_pool.convert_template(
-                                    expected_value
-                                ).value[0]
-                                if expected_value.value_type in {"number", "integer", "float"}:
-                                    expected_value = expected_value.value
-                                elif expected_value.value_type == "string":
-                                    expected_value = re.sub(r"[\r\n\t]+", " ", expected_value.text).strip()
-                                else:
-                                    raise ValueError("Invalid expected metadata value type")
-                        conditions.append(
-                            Condition(
-                                name=metadata_name,
-                                comparison_operator=condition.comparison_operator,
-                                value=expected_value,
-                            )
-                        )
-                        filters = DatasetRetrieval.process_metadata_filter_func(
-                            sequence,
-                            condition.comparison_operator,
-                            metadata_name,
-                            expected_value,
-                            filters,
-                        )
-                    metadata_condition = MetadataCondition(
-                        logical_operator=node_data.metadata_filtering_conditions.logical_operator,
-                        conditions=conditions,
+                    filters = DatasetRetrieval.process_metadata_filter_func(
+                        sequence,
+                        condition.comparison_operator,
+                        metadata_name,
+                        expected_value,
+                        filters,
                     )
-            case _:
-                raise ValueError("Invalid metadata filtering mode")
+                metadata_condition = MetadataCondition(
+                    logical_operator=node_data.metadata_filtering_conditions.logical_operator,
+                    conditions=conditions,
+                )
+        else:
+            raise ValueError("Invalid metadata filtering mode")
         if filters:
             if (
                 node_data.metadata_filtering_conditions

api/core/workflow/nodes/llm/node.py

@@ -322,8 +322,6 @@ class LLMNode(Node[LLMNodeData]):
                     outputs=outputs,
                     metadata={
                         WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS: usage.total_tokens,
-                        WorkflowNodeExecutionMetadataKey.PROMPT_TOKENS: usage.prompt_tokens,
-                        WorkflowNodeExecutionMetadataKey.COMPLETION_TOKENS: usage.completion_tokens,
                         WorkflowNodeExecutionMetadataKey.TOTAL_PRICE: usage.total_price,
                         WorkflowNodeExecutionMetadataKey.CURRENCY: usage.currency,
                     },

api/core/workflow/nodes/tool/tool_node.py

@@ -60,9 +60,7 @@ class ToolNode(Node[ToolNodeData]):
         tool_info = {
             "provider_type": self.node_data.provider_type.value,
             "provider_id": self.node_data.provider_id,
-            "tool_name": self.node_data.tool_name,
             "plugin_unique_identifier": self.node_data.plugin_unique_identifier,
-            "credential_id": self.node_data.credential_id,
         }
 
         # get tool runtime
@@ -107,20 +105,6 @@ class ToolNode(Node[ToolNodeData]):
         # get conversation id
         conversation_id = self.graph_runtime_state.variable_pool.get(["sys", SystemVariableKey.CONVERSATION_ID])
 
-        from core.tools.workflow_as_tool.tool import WorkflowTool
-
-        if isinstance(tool_runtime, WorkflowTool):
-            workflow_run_id_var = self.graph_runtime_state.variable_pool.get(
-                ["sys", SystemVariableKey.WORKFLOW_EXECUTION_ID]
-            )
-            tool_runtime.parent_trace_context = {
-                "trace_id": str(workflow_run_id_var.text) if workflow_run_id_var else "",
-                "parent_node_execution_id": self.execution_id,
-                "parent_workflow_run_id": str(workflow_run_id_var.text) if workflow_run_id_var else "",
-                "parent_app_id": self.app_id,
-                "parent_conversation_id": conversation_id.text if conversation_id else None,
-            }
-
         try:
             message_stream = ToolEngine.generic_invoke(
                 tool=tool_runtime,
@@ -447,8 +431,6 @@ class ToolNode(Node[ToolNodeData]):
         }
         if isinstance(usage.total_tokens, int) and usage.total_tokens > 0:
             metadata[WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS] = usage.total_tokens
-            metadata[WorkflowNodeExecutionMetadataKey.PROMPT_TOKENS] = usage.prompt_tokens
-            metadata[WorkflowNodeExecutionMetadataKey.COMPLETION_TOKENS] = usage.completion_tokens
             metadata[WorkflowNodeExecutionMetadataKey.TOTAL_PRICE] = usage.total_price
             metadata[WorkflowNodeExecutionMetadataKey.CURRENCY] = usage.currency
 
@@ -500,17 +482,16 @@ class ToolNode(Node[ToolNodeData]):
         result = {}
         for parameter_name in typed_node_data.tool_parameters:
             input = typed_node_data.tool_parameters[parameter_name]
-            match input.type:
-                case "mixed":
-                    assert isinstance(input.value, str)
-                    selectors = VariableTemplateParser(input.value).extract_variable_selectors()
-                    for selector in selectors:
-                        result[selector.variable] = selector.value_selector
-                case "variable":
-                    selector_key = ".".join(input.value)
-                    result[f"#{selector_key}#"] = input.value
-                case "constant":
-                    pass
+            if input.type == "mixed":
+                assert isinstance(input.value, str)
+                selectors = VariableTemplateParser(input.value).extract_variable_selectors()
+                for selector in selectors:
+                    result[selector.variable] = selector.value_selector
+            elif input.type == "variable":
+                selector_key = ".".join(input.value)
+                result[f"#{selector_key}#"] = input.value
+            elif input.type == "constant":
+                pass
 
         result = {node_id + "." + key: value for key, value in result.items()}
 

api/enterprise/telemetry/DATA_DICTIONARY.md

@@ -1,522 +0,0 @@
-# Dify Enterprise Telemetry Data Dictionary
-
-Quick reference for all telemetry signals emitted by Dify Enterprise. For configuration and architecture details, see [README.md](./README.md).
-
-## Resource Attributes
-
-Attached to every signal (Span, Metric, Log).
-
-| Attribute | Type | Example |
-|-----------|------|---------|
-| `service.name` | string | `dify` |
-| `host.name` | string | `dify-api-7f8b` |
-
-## Traces (Spans)
-
-### `dify.workflow.run`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.trace_id` | string | Business trace ID (Workflow Run ID) |
-| `dify.tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.workflow.id` | string | Workflow definition ID |
-| `dify.workflow.run_id` | string | Unique ID for this run |
-| `dify.workflow.status` | string | `succeeded`, `failed`, `stopped`, etc. |
-| `dify.workflow.error` | string | Error message if failed |
-| `dify.workflow.elapsed_time` | float | Total execution time (seconds) |
-| `dify.invoke_from` | string | `api`, `webapp`, `debug` |
-| `dify.conversation.id` | string | Conversation ID (optional) |
-| `dify.message.id` | string | Message ID (optional) |
-| `dify.invoked_by` | string | User ID who triggered the run |
-| `gen_ai.usage.total_tokens` | int | Total tokens across all nodes (optional) |
-| `gen_ai.user.id` | string | End-user identifier (optional) |
-| `dify.parent.trace_id` | string | Parent workflow trace ID (optional) |
-| `dify.parent.workflow.run_id` | string | Parent workflow run ID (optional) |
-| `dify.parent.node.execution_id` | string | Parent node execution ID (optional) |
-| `dify.parent.app.id` | string | Parent app ID (optional) |
-
-### `dify.node.execution`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.trace_id` | string | Business trace ID |
-| `dify.tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.workflow.id` | string | Workflow definition ID |
-| `dify.workflow.run_id` | string | Workflow Run ID |
-| `dify.message.id` | string | Message ID (optional) |
-| `dify.conversation.id` | string | Conversation ID (optional) |
-| `dify.node.execution_id` | string | Unique node execution ID |
-| `dify.node.id` | string | Node ID in workflow graph |
-| `dify.node.type` | string | Node type (see appendix) |
-| `dify.node.title` | string | Display title |
-| `dify.node.status` | string | `succeeded`, `failed` |
-| `dify.node.error` | string | Error message if failed |
-| `dify.node.elapsed_time` | float | Execution time (seconds) |
-| `dify.node.index` | int | Execution order index |
-| `dify.node.predecessor_node_id` | string | Triggering node ID |
-| `dify.node.iteration_id` | string | Iteration ID (optional) |
-| `dify.node.loop_id` | string | Loop ID (optional) |
-| `dify.node.parallel_id` | string | Parallel branch ID (optional) |
-| `dify.node.invoked_by` | string | User ID who triggered execution |
-| `gen_ai.usage.input_tokens` | int | Prompt tokens (LLM nodes only) |
-| `gen_ai.usage.output_tokens` | int | Completion tokens (LLM nodes only) |
-| `gen_ai.usage.total_tokens` | int | Total tokens (LLM nodes only) |
-| `gen_ai.request.model` | string | LLM model name (LLM nodes only) |
-| `gen_ai.provider.name` | string | LLM provider name (LLM nodes only) |
-| `gen_ai.user.id` | string | End-user identifier (optional) |
-
-### `dify.node.execution.draft`
-
-Same attributes as `dify.node.execution`. Emitted during Preview/Debug runs.
-
-## Counters
-
-All counters are cumulative and emitted at 100% accuracy.
-
-### Token Counters
-
-| Metric | Unit | Description |
-|--------|------|-------------|
-| `dify.tokens.total` | `{token}` | Total tokens consumed |
-| `dify.tokens.input` | `{token}` | Input (prompt) tokens |
-| `dify.tokens.output` | `{token}` | Output (completion) tokens |
-
-**Labels:**
-- `tenant_id`, `app_id`, `operation_type`, `model_provider`, `model_name`, `node_type` (if node_execution)
-
-⚠️ **Warning:** `dify.tokens.total` at workflow level includes all node tokens. Filter by `operation_type` to avoid double-counting.
-
-#### Token Hierarchy & Query Patterns
-
-Token metrics are emitted at multiple layers. Understanding the hierarchy prevents double-counting:
-
-```
-App-level total
-├── workflow          ← sum of all node_execution tokens (DO NOT add both)
-│   └── node_execution ← per-node breakdown
-├── message           ← independent (non-workflow chat apps only)
-├── rule_generate     ← independent helper LLM call
-├── code_generate     ← independent helper LLM call
-├── structured_output ← independent helper LLM call
-└── instruction_modify← independent helper LLM call
-```
-
-**Key rule:** `workflow` tokens already include all `node_execution` tokens. Never sum both.
-
-**Available labels on token metrics:** `tenant_id`, `app_id`, `operation_type`, `model_provider`, `model_name`, `node_type`.
-App name is only available on span attributes (`dify.app.name`), not metric labels — use `app_id` for metric queries.
-
-**Common queries** (PromQL):
-
-```promql
-# ── Totals ──────────────────────────────────────────────────
-# App-level total (exclude node_execution to avoid double-counting)
-sum by (app_id) (dify_tokens_total{operation_type!="node_execution"})
-
-# Single app total
-sum (dify_tokens_total{app_id="<app_id>", operation_type!="node_execution"})
-
-# Per-tenant totals
-sum by (tenant_id) (dify_tokens_total{operation_type!="node_execution"})
-
-# ── Drill-down ──────────────────────────────────────────────
-# Workflow-level tokens for an app
-sum (dify_tokens_total{app_id="<app_id>", operation_type="workflow"})
-
-# Node-level breakdown within an app
-sum by (node_type) (dify_tokens_total{app_id="<app_id>", operation_type="node_execution"})
-
-# Model breakdown for an app
-sum by (model_provider, model_name) (dify_tokens_total{app_id="<app_id>"})
-
-# Input vs output per model
-sum by (model_name) (dify_tokens_input_total{app_id="<app_id>"})
-sum by (model_name) (dify_tokens_output_total{app_id="<app_id>"})
-
-# ── Rates ───────────────────────────────────────────────────
-# Token consumption rate (per hour)
-sum(rate(dify_tokens_total{operation_type!="node_execution"}[1h]))
-
-# Per-app consumption rate
-sum by (app_id) (rate(dify_tokens_total{operation_type!="node_execution"}[1h]))
-```
-
-**Finding `app_id` from app name** (trace query — Tempo / Jaeger):
-
-```
-{ resource.dify.app.name = "My Chatbot" } | select(resource.dify.app.id)
-```
-
-### Request Counters
-
-| Metric | Unit | Description |
-|--------|------|-------------|
-| `dify.requests.total` | `{request}` | Total operations count |
-
-**Labels by type:**
-
-| `type` | Additional Labels |
-|--------|-------------------|
-| `workflow` | `tenant_id`, `app_id`, `status`, `invoke_from` |
-| `node` | `tenant_id`, `app_id`, `node_type`, `model_provider`, `model_name`, `status` |
-| `draft_node` | `tenant_id`, `app_id`, `node_type`, `model_provider`, `model_name`, `status` |
-| `message` | `tenant_id`, `app_id`, `model_provider`, `model_name`, `status`, `invoke_from` |
-| `tool` | `tenant_id`, `app_id`, `tool_name` |
-| `moderation` | `tenant_id`, `app_id` |
-| `suggested_question` | `tenant_id`, `app_id`, `model_provider`, `model_name` |
-| `dataset_retrieval` | `tenant_id`, `app_id` |
-| `generate_name` | `tenant_id`, `app_id` |
-| `prompt_generation` | `tenant_id`, `app_id`, `operation_type`, `model_provider`, `model_name`, `status` |
-
-### Error Counters
-
-| Metric | Unit | Description |
-|--------|------|-------------|
-| `dify.errors.total` | `{error}` | Total failed operations |
-
-**Labels by type:**
-
-| `type` | Additional Labels |
-|--------|-------------------|
-| `workflow` | `tenant_id`, `app_id` |
-| `node` | `tenant_id`, `app_id`, `node_type`, `model_provider`, `model_name` |
-| `draft_node` | `tenant_id`, `app_id`, `node_type`, `model_provider`, `model_name` |
-| `message` | `tenant_id`, `app_id`, `model_provider`, `model_name` |
-| `tool` | `tenant_id`, `app_id`, `tool_name` |
-| `prompt_generation` | `tenant_id`, `app_id`, `operation_type`, `model_provider`, `model_name` |
-
-### Other Counters
-
-| Metric | Unit | Labels |
-|--------|------|--------|
-| `dify.feedback.total` | `{feedback}` | `tenant_id`, `app_id`, `rating` |
-| `dify.dataset.retrievals.total` | `{retrieval}` | `tenant_id`, `app_id`, `dataset_id`, `embedding_model_provider`, `embedding_model`, `rerank_model_provider`, `rerank_model` |
-| `dify.app.created.total` | `{app}` | `tenant_id`, `app_id`, `mode` |
-| `dify.app.updated.total` | `{app}` | `tenant_id`, `app_id` |
-| `dify.app.deleted.total` | `{app}` | `tenant_id`, `app_id` |
-
-## Histograms
-
-| Metric | Unit | Labels |
-|--------|------|--------|
-| `dify.workflow.duration` | `s` | `tenant_id`, `app_id`, `status` |
-| `dify.node.duration` | `s` | `tenant_id`, `app_id`, `node_type`, `model_provider`, `model_name`, `plugin_name` |
-| `dify.message.duration` | `s` | `tenant_id`, `app_id`, `model_provider`, `model_name` |
-| `dify.message.time_to_first_token` | `s` | `tenant_id`, `app_id`, `model_provider`, `model_name` |
-| `dify.tool.duration` | `s` | `tenant_id`, `app_id`, `tool_name` |
-| `dify.prompt_generation.duration` | `s` | `tenant_id`, `app_id`, `operation_type`, `model_provider`, `model_name` |
-
-## Structured Logs
-
-### Span Companion Logs
-
-Logs that accompany spans. Signal type: `span_detail`
-
-#### `dify.workflow.run` Companion Log
-
-**Common attributes:** All span attributes (see Traces section) plus:
-
-| Additional Attribute | Type | Always Present | Description |
-|---------------------|------|----------------|-------------|
-| `dify.app.name` | string | No | Application display name |
-| `dify.workspace.name` | string | No | Workspace display name |
-| `dify.workflow.version` | string | Yes | Workflow definition version |
-| `dify.workflow.inputs` | string/JSON | Yes | Input parameters (content-gated) |
-| `dify.workflow.outputs` | string/JSON | Yes | Output results (content-gated) |
-| `dify.workflow.query` | string | No | User query text (content-gated) |
-
-**Event attributes:**
-- `dify.event.name`: `"dify.workflow.run"`
-- `dify.event.signal`: `"span_detail"`
-- `trace_id`, `span_id`, `tenant_id`, `user_id`
-
-#### `dify.node.execution` and `dify.node.execution.draft` Companion Logs
-
-**Common attributes:** All span attributes (see Traces section) plus:
-
-| Additional Attribute | Type | Always Present | Description |
-|---------------------|------|----------------|-------------|
-| `dify.app.name` | string | No | Application display name |
-| `dify.workspace.name` | string | No | Workspace display name |
-| `dify.invoke_from` | string | No | Invocation source |
-| `gen_ai.tool.name` | string | No | Tool name (tool nodes only) |
-| `dify.node.total_price` | float | No | Cost (LLM nodes only) |
-| `dify.node.currency` | string | No | Currency code (LLM nodes only) |
-| `dify.node.iteration_index` | int | No | Iteration index (iteration nodes) |
-| `dify.node.loop_index` | int | No | Loop index (loop nodes) |
-| `dify.plugin.name` | string | No | Plugin name (tool/knowledge nodes) |
-| `dify.credential.name` | string | No | Credential name (plugin nodes) |
-| `dify.credential.id` | string | No | Credential ID (plugin nodes) |
-| `dify.dataset.ids` | JSON array | No | Dataset IDs (knowledge nodes) |
-| `dify.dataset.names` | JSON array | No | Dataset names (knowledge nodes) |
-| `dify.node.inputs` | string/JSON | Yes | Node inputs (content-gated) |
-| `dify.node.outputs` | string/JSON | Yes | Node outputs (content-gated) |
-| `dify.node.process_data` | string/JSON | No | Processing data (content-gated) |
-
-**Event attributes:**
-- `dify.event.name`: `"dify.node.execution"` or `"dify.node.execution.draft"`
-- `dify.event.signal`: `"span_detail"`
-- `trace_id`, `span_id`, `tenant_id`, `user_id`
-
-### Standalone Logs
-
-Logs without structural spans. Signal type: `metric_only`
-
-#### `dify.message.run`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.message.run"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `trace_id` | string | OTEL trace ID (32-char hex) |
-| `span_id` | string | OTEL span ID (16-char hex) |
-| `tenant_id` | string | Tenant identifier |
-| `user_id` | string | User identifier (optional) |
-| `dify.app_id` | string | Application identifier |
-| `dify.message.id` | string | Message identifier |
-| `dify.conversation.id` | string | Conversation ID (optional) |
-| `dify.workflow.run_id` | string | Workflow run ID (optional) |
-| `dify.invoke_from` | string | `service-api`, `web-app`, `debugger`, `explore` |
-| `gen_ai.provider.name` | string | LLM provider |
-| `gen_ai.request.model` | string | LLM model |
-| `gen_ai.usage.input_tokens` | int | Input tokens |
-| `gen_ai.usage.output_tokens` | int | Output tokens |
-| `gen_ai.usage.total_tokens` | int | Total tokens |
-| `dify.message.status` | string | `succeeded`, `failed` |
-| `dify.message.error` | string | Error message (if failed) |
-| `dify.message.duration` | float | Duration (seconds) |
-| `dify.message.time_to_first_token` | float | TTFT (seconds) |
-| `dify.message.inputs` | string/JSON | Inputs (content-gated) |
-| `dify.message.outputs` | string/JSON | Outputs (content-gated) |
-
-#### `dify.tool.execution`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.tool.execution"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `trace_id` | string | OTEL trace ID |
-| `span_id` | string | OTEL span ID |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.message.id` | string | Message identifier |
-| `dify.tool.name` | string | Tool name |
-| `dify.tool.duration` | float | Duration (seconds) |
-| `dify.tool.status` | string | `succeeded`, `failed` |
-| `dify.tool.error` | string | Error message (if failed) |
-| `dify.tool.inputs` | string/JSON | Inputs (content-gated) |
-| `dify.tool.outputs` | string/JSON | Outputs (content-gated) |
-| `dify.tool.parameters` | string/JSON | Parameters (content-gated) |
-| `dify.tool.config` | string/JSON | Configuration (content-gated) |
-
-#### `dify.moderation.check`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.moderation.check"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `trace_id` | string | OTEL trace ID |
-| `span_id` | string | OTEL span ID |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.message.id` | string | Message identifier |
-| `dify.moderation.type` | string | `input`, `output` |
-| `dify.moderation.action` | string | `pass`, `block`, `flag` |
-| `dify.moderation.flagged` | boolean | Whether flagged |
-| `dify.moderation.categories` | JSON array | Flagged categories |
-| `dify.moderation.query` | string | Content (content-gated) |
-
-#### `dify.suggested_question.generation`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.suggested_question.generation"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `trace_id` | string | OTEL trace ID |
-| `span_id` | string | OTEL span ID |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.message.id` | string | Message identifier |
-| `dify.suggested_question.count` | int | Number of questions |
-| `dify.suggested_question.duration` | float | Duration (seconds) |
-| `dify.suggested_question.status` | string | `succeeded`, `failed` |
-| `dify.suggested_question.error` | string | Error message (if failed) |
-| `dify.suggested_question.questions` | JSON array | Questions (content-gated) |
-
-#### `dify.dataset.retrieval`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.dataset.retrieval"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `trace_id` | string | OTEL trace ID |
-| `span_id` | string | OTEL span ID |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.message.id` | string | Message identifier |
-| `dify.dataset.id` | string | Dataset identifier |
-| `dify.dataset.name` | string | Dataset name |
-| `dify.dataset.embedding_providers` | JSON array | Embedding model providers (one per dataset) |
-| `dify.dataset.embedding_models` | JSON array | Embedding models (one per dataset) |
-| `dify.retrieval.rerank_provider` | string | Rerank model provider |
-| `dify.retrieval.rerank_model` | string | Rerank model name |
-| `dify.retrieval.query` | string | Search query (content-gated) |
-| `dify.retrieval.document_count` | int | Documents retrieved |
-| `dify.retrieval.duration` | float | Duration (seconds) |
-| `dify.retrieval.status` | string | `succeeded`, `failed` |
-| `dify.retrieval.error` | string | Error message (if failed) |
-| `dify.dataset.documents` | JSON array | Documents (content-gated) |
-
-#### `dify.generate_name.execution`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.generate_name.execution"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `trace_id` | string | OTEL trace ID |
-| `span_id` | string | OTEL span ID |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.conversation.id` | string | Conversation identifier |
-| `dify.generate_name.duration` | float | Duration (seconds) |
-| `dify.generate_name.status` | string | `succeeded`, `failed` |
-| `dify.generate_name.error` | string | Error message (if failed) |
-| `dify.generate_name.inputs` | string/JSON | Inputs (content-gated) |
-| `dify.generate_name.outputs` | string | Generated name (content-gated) |
-
-#### `dify.prompt_generation.execution`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.prompt_generation.execution"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `trace_id` | string | OTEL trace ID |
-| `span_id` | string | OTEL span ID |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.prompt_generation.operation_type` | string | Operation type (see appendix) |
-| `gen_ai.provider.name` | string | LLM provider |
-| `gen_ai.request.model` | string | LLM model |
-| `gen_ai.usage.input_tokens` | int | Input tokens |
-| `gen_ai.usage.output_tokens` | int | Output tokens |
-| `gen_ai.usage.total_tokens` | int | Total tokens |
-| `dify.prompt_generation.duration` | float | Duration (seconds) |
-| `dify.prompt_generation.status` | string | `succeeded`, `failed` |
-| `dify.prompt_generation.error` | string | Error message (if failed) |
-| `dify.prompt_generation.instruction` | string | Instruction (content-gated) |
-| `dify.prompt_generation.output` | string/JSON | Output (content-gated) |
-
-#### `dify.app.created`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.app.created"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.app.mode` | string | `chat`, `completion`, `agent-chat`, `workflow` |
-| `dify.app.created_at` | string | Timestamp (ISO 8601) |
-
-#### `dify.app.updated`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.app.updated"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.app.updated_at` | string | Timestamp (ISO 8601) |
-
-#### `dify.app.deleted`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.app.deleted"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.app.deleted_at` | string | Timestamp (ISO 8601) |
-
-#### `dify.feedback.created`
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.feedback.created"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `trace_id` | string | OTEL trace ID |
-| `span_id` | string | OTEL span ID |
-| `tenant_id` | string | Tenant identifier |
-| `dify.app_id` | string | Application identifier |
-| `dify.message.id` | string | Message identifier |
-| `dify.feedback.rating` | string | `like`, `dislike`, `null` |
-| `dify.feedback.content` | string | Feedback text (content-gated) |
-| `dify.feedback.created_at` | string | Timestamp (ISO 8601) |
-
-#### `dify.telemetry.rehydration_failed`
-
-Diagnostic event for telemetry system health monitoring.
-
-| Attribute | Type | Description |
-|-----------|------|-------------|
-| `dify.event.name` | string | `"dify.telemetry.rehydration_failed"` |
-| `dify.event.signal` | string | `"metric_only"` |
-| `tenant_id` | string | Tenant identifier |
-| `dify.telemetry.error` | string | Error message |
-| `dify.telemetry.payload_type` | string | Payload type (see appendix) |
-| `dify.telemetry.correlation_id` | string | Correlation ID |
-
-## Content-Gated Attributes
-
-When `ENTERPRISE_INCLUDE_CONTENT=false`, these attributes are replaced with reference strings (`ref:{id_type}={uuid}`).
-
-| Attribute | Signal |
-|-----------|--------|
-| `dify.workflow.inputs` | `dify.workflow.run` |
-| `dify.workflow.outputs` | `dify.workflow.run` |
-| `dify.workflow.query` | `dify.workflow.run` |
-| `dify.node.inputs` | `dify.node.execution` |
-| `dify.node.outputs` | `dify.node.execution` |
-| `dify.node.process_data` | `dify.node.execution` |
-| `dify.message.inputs` | `dify.message.run` |
-| `dify.message.outputs` | `dify.message.run` |
-| `dify.tool.inputs` | `dify.tool.execution` |
-| `dify.tool.outputs` | `dify.tool.execution` |
-| `dify.tool.parameters` | `dify.tool.execution` |
-| `dify.tool.config` | `dify.tool.execution` |
-| `dify.moderation.query` | `dify.moderation.check` |
-| `dify.suggested_question.questions` | `dify.suggested_question.generation` |
-| `dify.retrieval.query` | `dify.dataset.retrieval` |
-| `dify.dataset.documents` | `dify.dataset.retrieval` |
-| `dify.generate_name.inputs` | `dify.generate_name.execution` |
-| `dify.generate_name.outputs` | `dify.generate_name.execution` |
-| `dify.prompt_generation.instruction` | `dify.prompt_generation.execution` |
-| `dify.prompt_generation.output` | `dify.prompt_generation.execution` |
-| `dify.feedback.content` | `dify.feedback.created` |
-
-## Appendix
-
-### Operation Types
-
-- `workflow`, `node_execution`, `message`, `rule_generate`, `code_generate`, `structured_output`, `instruction_modify`
-
-### Node Types
-
-- `start`, `end`, `answer`, `llm`, `knowledge-retrieval`, `knowledge-index`, `if-else`, `code`, `template-transform`, `question-classifier`, `http-request`, `tool`, `datasource`, `variable-aggregator`, `loop`, `iteration`, `parameter-extractor`, `assigner`, `document-extractor`, `list-operator`, `agent`, `trigger-webhook`, `trigger-schedule`, `trigger-plugin`, `human-input`
-
-### Workflow Statuses
-
-- `running`, `succeeded`, `failed`, `stopped`, `partial-succeeded`, `paused`
-
-### Payload Types
-
-- `workflow`, `node`, `message`, `tool`, `moderation`, `suggested_question`, `dataset_retrieval`, `generate_name`, `prompt_generation`, `app`, `feedback`
-
-### Null Value Behavior
-
-**Spans:** Attributes with `null` values are omitted.
-
-**Logs:** Attributes with `null` values appear as `null` in JSON.
-
-**Content-Gated:** Replaced with reference strings, not set to `null`.

api/enterprise/telemetry/README.md

@@ -1,116 +0,0 @@
-# Dify Enterprise Telemetry
-
-This document provides an overview of the Dify Enterprise OpenTelemetry (OTEL) exporter and how to configure it for integration with observability stacks like Prometheus, Grafana, Jaeger, or Honeycomb.
-
-## Overview
-
-Dify Enterprise uses a "slim span + rich companion log" architecture to provide high-fidelity observability without overwhelming trace storage.
-
-- **Traces (Spans)**: Capture the structure, identity, and timing of high-level operations (Workflows and Nodes).
-- **Structured Logs**: Provide deep context (inputs, outputs, metadata) for every event, correlated to spans via `trace_id` and `span_id`.
-- **Metrics**: Provide 100% accurate counters and histograms for usage, performance, and error tracking.
-
-### Signal Architecture
-
-```mermaid
-graph TD
-    A[Workflow Run] -->|Span| B(dify.workflow.run)
-    A -->|Log| C(dify.workflow.run detail)
-    B ---|trace_id| C
-    
-    D[Node Execution] -->|Span| E(dify.node.execution)
-    D -->|Log| F(dify.node.execution detail)
-    E ---|span_id| F
-    
-    G[Message/Tool/etc] -->|Log| H(dify.* event)
-    G -->|Metric| I(dify.* counter/histogram)
-```
-
-## Configuration
-
-The Enterprise OTEL exporter is configured via environment variables.
-
-| Variable | Description | Default |
-|----------|-------------|---------|
-| `ENTERPRISE_ENABLED` | Master switch for all enterprise features. | `false` |
-| `ENTERPRISE_TELEMETRY_ENABLED` | Master switch for enterprise telemetry. | `false` |
-| `ENTERPRISE_OTLP_ENDPOINT` | OTLP collector endpoint (e.g., `http://otel-collector:4318`). | - |
-| `ENTERPRISE_OTLP_HEADERS` | Custom headers for OTLP requests (e.g., `x-scope-orgid=tenant1`). | - |
-| `ENTERPRISE_OTLP_PROTOCOL` | OTLP transport protocol (`http` or `grpc`). | `http` |
-| `ENTERPRISE_OTLP_API_KEY` | Bearer token for authentication. | - |
-| `ENTERPRISE_INCLUDE_CONTENT` | Whether to include sensitive content (inputs/outputs) in logs. | `true` |
-| `ENTERPRISE_SERVICE_NAME` | Service name reported to OTEL. | `dify` |
-| `ENTERPRISE_OTEL_SAMPLING_RATE` | Sampling rate for traces (0.0 to 1.0). Metrics are always 100%. | `1.0` |
-
-## Correlation Model
-
-Dify uses deterministic ID generation to ensure signals are correlated across different services and asynchronous tasks.
-
-### ID Generation Rules
-- `trace_id`: Derived from the correlation ID (workflow_run_id or node_execution_id for drafts) using `int(UUID(correlation_id))`
-- `span_id`: Derived from the source ID using `SHA256(source_id)[:8]`
-
-### Scenario A: Simple Workflow
-A single workflow run with multiple nodes. All spans and logs share the same `trace_id` (derived from `workflow_run_id`).
-
-```
-trace_id = UUID(workflow_run_id)
-├── [root span] dify.workflow.run (span_id = hash(workflow_run_id))
-│   ├── [child] dify.node.execution - "Start" (span_id = hash(node_exec_id_1))
-│   ├── [child] dify.node.execution - "LLM" (span_id = hash(node_exec_id_2))
-│   └── [child] dify.node.execution - "End" (span_id = hash(node_exec_id_3))
-```
-
-### Scenario B: Nested Sub-Workflow
-A workflow calling another workflow via a Tool or Sub-workflow node. The child workflow's spans are linked to the parent via `parent_span_id`. Both workflows share the same trace_id.
-
-```
-trace_id = UUID(outer_workflow_run_id)     ← shared across both workflows
-├── [root] dify.workflow.run (outer) (span_id = hash(outer_workflow_run_id))
-│   ├── dify.node.execution - "Start Node"
-│   ├── dify.node.execution - "Tool Node" (triggers sub-workflow)
-│   │   └── [child] dify.workflow.run (inner) (span_id = hash(inner_workflow_run_id))
-│   │       ├── dify.node.execution - "Inner Start"
-│   │       └── dify.node.execution - "Inner End"
-│   └── dify.node.execution - "End Node"
-```
-
-**Key attributes for nested workflows:**
-- Inner workflow's `dify.parent.trace_id` = outer `workflow_run_id`
-- Inner workflow's `dify.parent.node.execution_id` = tool node's `execution_id`
-- Inner workflow's `dify.parent.workflow.run_id` = outer `workflow_run_id`
-- Inner workflow's `dify.parent.app.id` = outer `app_id`
-
-### Scenario C: Draft Node Execution
-A single node run in isolation (debugger/preview mode). It creates its own trace where the node span is the root.
-
-```
-trace_id = UUID(node_execution_id)   ← own trace, NOT part of any workflow
-└── dify.node.execution.draft (span_id = hash(node_execution_id))
-```
-
-**Key difference:** Draft executions use `node_execution_id` as the correlation_id, so they are NOT children of any workflow trace.
-
-## Content Gating
-
-When `ENTERPRISE_INCLUDE_CONTENT` is set to `false`, sensitive content attributes (inputs, outputs, queries) are replaced with reference strings (e.g., `ref:workflow_run_id=...`) to prevent data leakage to the OTEL collector.
-
-**Reference String Format:**
-
-```
-ref:{id_type}={uuid}
-```
-
-**Examples:**
-
-```
-ref:workflow_run_id=550e8400-e29b-41d4-a716-446655440000
-ref:node_execution_id=660e8400-e29b-41d4-a716-446655440001
-ref:message_id=770e8400-e29b-41d4-a716-446655440002
-```
-
-To retrieve actual content when gating is enabled, query the Dify database using the provided UUID.
-
-## Reference
-
-For a complete list of telemetry signals, attributes, and data structures, see [DATA_DICTIONARY.md](./DATA_DICTIONARY.md).

api/enterprise/telemetry/contracts.py

@@ -1,73 +0,0 @@
-"""Telemetry gateway contracts and data structures.
-
-This module defines the envelope format for telemetry events and the routing
-configuration that determines how each event type is processed.
-"""
-
-from __future__ import annotations
-
-from enum import StrEnum
-from typing import Any
-
-from pydantic import BaseModel, ConfigDict
-
-
-class TelemetryCase(StrEnum):
-    """Enumeration of all known telemetry event cases."""
-
-    WORKFLOW_RUN = "workflow_run"
-    NODE_EXECUTION = "node_execution"
-    DRAFT_NODE_EXECUTION = "draft_node_execution"
-    MESSAGE_RUN = "message_run"
-    TOOL_EXECUTION = "tool_execution"
-    MODERATION_CHECK = "moderation_check"
-    SUGGESTED_QUESTION = "suggested_question"
-    DATASET_RETRIEVAL = "dataset_retrieval"
-    GENERATE_NAME = "generate_name"
-    PROMPT_GENERATION = "prompt_generation"
-    APP_CREATED = "app_created"
-    APP_UPDATED = "app_updated"
-    APP_DELETED = "app_deleted"
-    FEEDBACK_CREATED = "feedback_created"
-
-
-class SignalType(StrEnum):
-    """Signal routing type for telemetry cases."""
-
-    TRACE = "trace"
-    METRIC_LOG = "metric_log"
-
-
-class CaseRoute(BaseModel):
-    """Routing configuration for a telemetry case.
-
-    Attributes:
-        signal_type: The type of signal (trace or metric_log).
-        ce_eligible: Whether this case is eligible for community edition tracing.
-    """
-
-    signal_type: SignalType
-    ce_eligible: bool
-
-
-class TelemetryEnvelope(BaseModel):
-    """Envelope for telemetry events.
-
-    Attributes:
-        case: The telemetry case type.
-        tenant_id: The tenant identifier.
-        event_id: Unique event identifier for deduplication.
-        payload: The main event payload (inline for small payloads,
-            empty when offloaded to storage via ``payload_ref``).
-        metadata: Optional metadata dictionary.  When the gateway
-            offloads a large payload to object storage, this contains
-            ``{"payload_ref": "<storage_key>"}``.
-    """
-
-    model_config = ConfigDict(extra="forbid", use_enum_values=False)
-
-    case: TelemetryCase
-    tenant_id: str
-    event_id: str
-    payload: dict[str, Any]
-    metadata: dict[str, Any] | None = None

api/enterprise/telemetry/draft_trace.py

@@ -1,77 +0,0 @@
-from __future__ import annotations
-
-from collections.abc import Mapping
-from typing import Any
-
-from core.telemetry import TelemetryContext, TelemetryEvent, TraceTaskName
-from core.telemetry import emit as telemetry_emit
-from core.workflow.enums import WorkflowNodeExecutionMetadataKey
-from models.workflow import WorkflowNodeExecutionModel
-
-
-def enqueue_draft_node_execution_trace(
-    *,
-    execution: WorkflowNodeExecutionModel,
-    outputs: Mapping[str, Any] | None,
-    workflow_execution_id: str | None,
-    user_id: str,
-) -> None:
-    node_data = _build_node_execution_data(
-        execution=execution,
-        outputs=outputs,
-        workflow_execution_id=workflow_execution_id,
-    )
-    telemetry_emit(
-        TelemetryEvent(
-            name=TraceTaskName.DRAFT_NODE_EXECUTION_TRACE,
-            context=TelemetryContext(
-                tenant_id=execution.tenant_id,
-                user_id=user_id,
-                app_id=execution.app_id,
-            ),
-            payload={"node_execution_data": node_data},
-        )
-    )
-
-
-def _build_node_execution_data(
-    *,
-    execution: WorkflowNodeExecutionModel,
-    outputs: Mapping[str, Any] | None,
-    workflow_execution_id: str | None,
-) -> dict[str, Any]:
-    metadata = execution.execution_metadata_dict
-    node_outputs = outputs if outputs is not None else execution.outputs_dict
-    execution_id = workflow_execution_id or execution.workflow_run_id or execution.id
-
-    return {
-        "workflow_id": execution.workflow_id,
-        "workflow_execution_id": execution_id,
-        "tenant_id": execution.tenant_id,
-        "app_id": execution.app_id,
-        "node_execution_id": execution.id,
-        "node_id": execution.node_id,
-        "node_type": execution.node_type,
-        "title": execution.title,
-        "status": execution.status,
-        "error": execution.error,
-        "elapsed_time": execution.elapsed_time,
-        "index": execution.index,
-        "predecessor_node_id": execution.predecessor_node_id,
-        "created_at": execution.created_at,
-        "finished_at": execution.finished_at,
-        "total_tokens": metadata.get(WorkflowNodeExecutionMetadataKey.TOTAL_TOKENS, 0),
-        "total_price": metadata.get(WorkflowNodeExecutionMetadataKey.TOTAL_PRICE, 0.0),
-        "currency": metadata.get(WorkflowNodeExecutionMetadataKey.CURRENCY),
-        "tool_name": (metadata.get(WorkflowNodeExecutionMetadataKey.TOOL_INFO) or {}).get("tool_name")
-        if isinstance(metadata.get(WorkflowNodeExecutionMetadataKey.TOOL_INFO), dict)
-        else None,
-        "iteration_id": metadata.get(WorkflowNodeExecutionMetadataKey.ITERATION_ID),
-        "iteration_index": metadata.get(WorkflowNodeExecutionMetadataKey.ITERATION_INDEX),
-        "loop_id": metadata.get(WorkflowNodeExecutionMetadataKey.LOOP_ID),
-        "loop_index": metadata.get(WorkflowNodeExecutionMetadataKey.LOOP_INDEX),
-        "parallel_id": metadata.get(WorkflowNodeExecutionMetadataKey.PARALLEL_ID),
-        "node_inputs": execution.inputs_dict,
-        "node_outputs": node_outputs,
-        "process_data": execution.process_data_dict,
-    }

api/enterprise/telemetry/enterprise_trace.py

@@ -1,938 +0,0 @@
-"""Enterprise trace handler — duck-typed, NOT a BaseTraceInstance subclass.
-
-Invoked directly in the Celery task, not through OpsTraceManager dispatch.
-Only requires a matching ``trace(trace_info)`` method signature.
-
-Signal strategy:
-- **Traces (spans)**: workflow run, node execution, draft node execution only.
-- **Metrics + structured logs**: all other event types.
-
-Token metric labels (unified structure):
-All token metrics (dify.tokens.input, dify.tokens.output, dify.tokens.total) use the
-same label set for consistent filtering and aggregation:
-- tenant_id: Tenant identifier
-- app_id: Application identifier
-- operation_type: Source of token usage (workflow | node_execution | message | rule_generate | etc.)
-- model_provider: LLM provider name (empty string if not applicable)
-- model_name: LLM model name (empty string if not applicable)
-- node_type: Workflow node type (empty string if not node_execution)
-
-This unified structure allows filtering by operation_type to separate:
-- Workflow-level aggregates (operation_type=workflow)
-- Individual node executions (operation_type=node_execution)
-- Direct message calls (operation_type=message)
-- Prompt generation operations (operation_type=rule_generate, code_generate, etc.)
-
-Without this, tokens are double-counted when querying totals (workflow totals include
-node totals, since workflow.total_tokens is the sum of all node tokens).
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-from typing import Any, cast
-
-from opentelemetry.util.types import AttributeValue
-
-from core.ops.entities.trace_entity import (
-    BaseTraceInfo,
-    DatasetRetrievalTraceInfo,
-    DraftNodeExecutionTrace,
-    GenerateNameTraceInfo,
-    MessageTraceInfo,
-    ModerationTraceInfo,
-    OperationType,
-    PromptGenerationTraceInfo,
-    SuggestedQuestionTraceInfo,
-    ToolTraceInfo,
-    WorkflowNodeTraceInfo,
-    WorkflowTraceInfo,
-)
-from enterprise.telemetry.entities import (
-    EnterpriseTelemetryCounter,
-    EnterpriseTelemetryEvent,
-    EnterpriseTelemetryHistogram,
-    EnterpriseTelemetrySpan,
-    TokenMetricLabels,
-)
-from enterprise.telemetry.telemetry_log import emit_metric_only_event, emit_telemetry_log
-
-logger = logging.getLogger(__name__)
-
-
-class EnterpriseOtelTrace:
-    """Duck-typed enterprise trace handler.
-
-    ``*_trace`` methods emit spans (workflow/node only) or structured logs
-    (all other events), plus metrics at 100 % accuracy.
-    """
-
-    def __init__(self) -> None:
-        from extensions.ext_enterprise_telemetry import get_enterprise_exporter
-
-        exporter = get_enterprise_exporter()
-        if exporter is None:
-            raise RuntimeError("EnterpriseOtelTrace instantiated but exporter is not initialized")
-        self._exporter = exporter
-
-    def trace(self, trace_info: BaseTraceInfo) -> None:
-        if isinstance(trace_info, WorkflowTraceInfo):
-            self._workflow_trace(trace_info)
-        elif isinstance(trace_info, MessageTraceInfo):
-            self._message_trace(trace_info)
-        elif isinstance(trace_info, ToolTraceInfo):
-            self._tool_trace(trace_info)
-        elif isinstance(trace_info, DraftNodeExecutionTrace):
-            self._draft_node_execution_trace(trace_info)
-        elif isinstance(trace_info, WorkflowNodeTraceInfo):
-            self._node_execution_trace(trace_info)
-        elif isinstance(trace_info, ModerationTraceInfo):
-            self._moderation_trace(trace_info)
-        elif isinstance(trace_info, SuggestedQuestionTraceInfo):
-            self._suggested_question_trace(trace_info)
-        elif isinstance(trace_info, DatasetRetrievalTraceInfo):
-            self._dataset_retrieval_trace(trace_info)
-        elif isinstance(trace_info, GenerateNameTraceInfo):
-            self._generate_name_trace(trace_info)
-        elif isinstance(trace_info, PromptGenerationTraceInfo):
-            self._prompt_generation_trace(trace_info)
-
-    def _common_attrs(self, trace_info: BaseTraceInfo) -> dict[str, Any]:
-        metadata = self._metadata(trace_info)
-        tenant_id, app_id, user_id = self._context_ids(trace_info, metadata)
-        return {
-            "dify.trace_id": trace_info.resolved_trace_id,
-            "dify.tenant_id": tenant_id,
-            "dify.app_id": app_id,
-            "dify.app.name": metadata.get("app_name"),
-            "dify.workspace.name": metadata.get("workspace_name"),
-            "gen_ai.user.id": user_id,
-            "dify.message.id": trace_info.message_id,
-        }
-
-    def _metadata(self, trace_info: BaseTraceInfo) -> dict[str, Any]:
-        return trace_info.metadata
-
-    def _context_ids(
-        self,
-        trace_info: BaseTraceInfo,
-        metadata: dict[str, Any],
-    ) -> tuple[str | None, str | None, str | None]:
-        tenant_id = getattr(trace_info, "tenant_id", None) or metadata.get("tenant_id")
-        app_id = getattr(trace_info, "app_id", None) or metadata.get("app_id")
-        user_id = getattr(trace_info, "user_id", None) or metadata.get("user_id")
-        return tenant_id, app_id, user_id
-
-    def _labels(self, **values: AttributeValue) -> dict[str, AttributeValue]:
-        return dict(values)
-
-    def _safe_payload_value(self, value: Any) -> str | dict[str, Any] | list[object] | None:
-        if isinstance(value, str):
-            return value
-        if isinstance(value, dict):
-            return cast(dict[str, Any], value)
-        if isinstance(value, list):
-            items: list[object] = []
-            for item in cast(list[object], value):
-                items.append(item)
-            return items
-        return None
-
-    def _content_or_ref(self, value: Any, ref: str) -> Any:
-        if self._exporter.include_content:
-            return self._maybe_json(value)
-        return ref
-
-    def _maybe_json(self, value: Any) -> str | None:
-        if value is None:
-            return None
-        if isinstance(value, str):
-            return value
-        try:
-            return json.dumps(value, default=str)
-        except (TypeError, ValueError):
-            return str(value)
-
-    # ------------------------------------------------------------------
-    # SPAN-emitting handlers (workflow, node execution, draft node)
-    # ------------------------------------------------------------------
-
-    def _workflow_trace(self, info: WorkflowTraceInfo) -> None:
-        metadata = self._metadata(info)
-        tenant_id, app_id, user_id = self._context_ids(info, metadata)
-        # -- Span attrs: identity + structure + status + timing + gen_ai scalars --
-        span_attrs: dict[str, Any] = {
-            "dify.trace_id": info.resolved_trace_id,
-            "dify.tenant_id": tenant_id,
-            "dify.app_id": app_id,
-            "dify.workflow.id": info.workflow_id,
-            "dify.workflow.run_id": info.workflow_run_id,
-            "dify.workflow.status": info.workflow_run_status,
-            "dify.workflow.error": info.error,
-            "dify.workflow.elapsed_time": info.workflow_run_elapsed_time,
-            "dify.invoke_from": metadata.get("triggered_from"),
-            "dify.conversation.id": info.conversation_id,
-            "dify.message.id": info.message_id,
-            "dify.invoked_by": info.invoked_by,
-            "gen_ai.usage.total_tokens": info.total_tokens,
-            "gen_ai.user.id": user_id,
-        }
-
-        trace_correlation_override, parent_span_id_source = info.resolved_parent_context
-
-        parent_ctx = metadata.get("parent_trace_context")
-        if isinstance(parent_ctx, dict):
-            parent_ctx_dict = cast(dict[str, Any], parent_ctx)
-            span_attrs["dify.parent.trace_id"] = parent_ctx_dict.get("trace_id")
-            span_attrs["dify.parent.node.execution_id"] = parent_ctx_dict.get("parent_node_execution_id")
-            span_attrs["dify.parent.workflow.run_id"] = parent_ctx_dict.get("parent_workflow_run_id")
-            span_attrs["dify.parent.app.id"] = parent_ctx_dict.get("parent_app_id")
-
-        self._exporter.export_span(
-            EnterpriseTelemetrySpan.WORKFLOW_RUN,
-            span_attrs,
-            correlation_id=info.workflow_run_id,
-            span_id_source=info.workflow_run_id,
-            start_time=info.start_time,
-            end_time=info.end_time,
-            trace_correlation_override=trace_correlation_override,
-            parent_span_id_source=parent_span_id_source,
-        )
-
-        # -- Companion log: ALL attrs (span + detail) for full picture --
-        log_attrs: dict[str, Any] = {**span_attrs}
-        log_attrs.update(
-            {
-                "dify.app.name": metadata.get("app_name"),
-                "dify.workspace.name": metadata.get("workspace_name"),
-                "gen_ai.user.id": user_id,
-                "gen_ai.usage.total_tokens": info.total_tokens,
-                "dify.workflow.version": info.workflow_run_version,
-            }
-        )
-
-        ref = f"ref:workflow_run_id={info.workflow_run_id}"
-        log_attrs["dify.workflow.inputs"] = self._content_or_ref(info.workflow_run_inputs, ref)
-        log_attrs["dify.workflow.outputs"] = self._content_or_ref(info.workflow_run_outputs, ref)
-        log_attrs["dify.workflow.query"] = self._content_or_ref(info.query, ref)
-
-        emit_telemetry_log(
-            event_name=EnterpriseTelemetryEvent.WORKFLOW_RUN,
-            attributes=log_attrs,
-            signal="span_detail",
-            trace_id_source=info.workflow_run_id,
-            span_id_source=info.workflow_run_id,
-            tenant_id=tenant_id,
-            user_id=user_id,
-        )
-
-        # -- Metrics --
-        labels = self._labels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-        )
-        token_labels = TokenMetricLabels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-            operation_type=OperationType.WORKFLOW,
-            model_provider="",
-            model_name="",
-            node_type="",
-        ).to_dict()
-        self._exporter.increment_counter(EnterpriseTelemetryCounter.TOKENS, info.total_tokens, token_labels)
-        if info.prompt_tokens is not None and info.prompt_tokens > 0:
-            self._exporter.increment_counter(EnterpriseTelemetryCounter.INPUT_TOKENS, info.prompt_tokens, token_labels)
-        if info.completion_tokens is not None and info.completion_tokens > 0:
-            self._exporter.increment_counter(
-                EnterpriseTelemetryCounter.OUTPUT_TOKENS, info.completion_tokens, token_labels
-            )
-        invoke_from = metadata.get("triggered_from", "")
-        self._exporter.increment_counter(
-            EnterpriseTelemetryCounter.REQUESTS,
-            1,
-            self._labels(
-                **labels,
-                type="workflow",
-                status=info.workflow_run_status,
-                invoke_from=invoke_from,
-            ),
-        )
-        # Prefer wall-clock timestamps over the elapsed_time field: elapsed_time defaults
-        # to 0 in the DB and can be stale if the Celery write races with the trace task.
-        # start_time = workflow_run.created_at, end_time = workflow_run.finished_at.
-        if info.start_time and info.end_time:
-            workflow_duration = (info.end_time - info.start_time).total_seconds()
-        elif info.workflow_run_elapsed_time:
-            workflow_duration = float(info.workflow_run_elapsed_time)
-        else:
-            workflow_duration = 0.0
-        self._exporter.record_histogram(
-            EnterpriseTelemetryHistogram.WORKFLOW_DURATION,
-            workflow_duration,
-            self._labels(
-                **labels,
-                status=info.workflow_run_status,
-            ),
-        )
-
-        if info.error:
-            self._exporter.increment_counter(
-                EnterpriseTelemetryCounter.ERRORS,
-                1,
-                self._labels(
-                    **labels,
-                    type="workflow",
-                ),
-            )
-
-    def _node_execution_trace(self, info: WorkflowNodeTraceInfo) -> None:
-        self._emit_node_execution_trace(info, EnterpriseTelemetrySpan.NODE_EXECUTION, "node")
-
-    def _draft_node_execution_trace(self, info: DraftNodeExecutionTrace) -> None:
-        self._emit_node_execution_trace(
-            info,
-            EnterpriseTelemetrySpan.DRAFT_NODE_EXECUTION,
-            "draft_node",
-            correlation_id_override=info.node_execution_id,
-            trace_correlation_override_param=info.workflow_run_id,
-        )
-
-    def _emit_node_execution_trace(
-        self,
-        info: WorkflowNodeTraceInfo,
-        span_name: EnterpriseTelemetrySpan,
-        request_type: str,
-        correlation_id_override: str | None = None,
-        trace_correlation_override_param: str | None = None,
-    ) -> None:
-        metadata = self._metadata(info)
-        tenant_id, app_id, user_id = self._context_ids(info, metadata)
-        # -- Span attrs: identity + structure + status + timing + gen_ai scalars --
-        span_attrs: dict[str, Any] = {
-            "dify.trace_id": info.resolved_trace_id,
-            "dify.tenant_id": tenant_id,
-            "dify.app_id": app_id,
-            "dify.workflow.id": info.workflow_id,
-            "dify.workflow.run_id": info.workflow_run_id,
-            "dify.message.id": info.message_id,
-            "dify.conversation.id": metadata.get("conversation_id"),
-            "dify.node.execution_id": info.node_execution_id,
-            "dify.node.id": info.node_id,
-            "dify.node.type": info.node_type,
-            "dify.node.title": info.title,
-            "dify.node.status": info.status,
-            "dify.node.error": info.error,
-            "dify.node.elapsed_time": info.elapsed_time,
-            "dify.node.index": info.index,
-            "dify.node.predecessor_node_id": info.predecessor_node_id,
-            "dify.node.iteration_id": info.iteration_id,
-            "dify.node.loop_id": info.loop_id,
-            "dify.node.parallel_id": info.parallel_id,
-            "dify.node.invoked_by": info.invoked_by,
-            "gen_ai.usage.input_tokens": info.prompt_tokens,
-            "gen_ai.usage.output_tokens": info.completion_tokens,
-            "gen_ai.usage.total_tokens": info.total_tokens,
-            "gen_ai.request.model": info.model_name,
-            "gen_ai.provider.name": info.model_provider,
-            "gen_ai.user.id": user_id,
-        }
-
-        resolved_override, _ = info.resolved_parent_context
-        trace_correlation_override = trace_correlation_override_param or resolved_override
-
-        effective_correlation_id = correlation_id_override or info.workflow_run_id
-        self._exporter.export_span(
-            span_name,
-            span_attrs,
-            correlation_id=effective_correlation_id,
-            span_id_source=info.node_execution_id,
-            start_time=info.start_time,
-            end_time=info.end_time,
-            trace_correlation_override=trace_correlation_override,
-        )
-
-        # -- Companion log: ALL attrs (span + detail) --
-        log_attrs: dict[str, Any] = {**span_attrs}
-        log_attrs.update(
-            {
-                "dify.app.name": metadata.get("app_name"),
-                "dify.workspace.name": metadata.get("workspace_name"),
-                "dify.invoke_from": metadata.get("invoke_from"),
-                "gen_ai.user.id": user_id,
-                "gen_ai.usage.total_tokens": info.total_tokens,
-                "dify.node.total_price": info.total_price,
-                "dify.node.currency": info.currency,
-                "gen_ai.provider.name": info.model_provider,
-                "gen_ai.request.model": info.model_name,
-                "gen_ai.tool.name": info.tool_name,
-                "dify.node.iteration_index": info.iteration_index,
-                "dify.node.loop_index": info.loop_index,
-                "dify.plugin.name": metadata.get("plugin_name"),
-                "dify.credential.name": metadata.get("credential_name"),
-                "dify.credential.id": metadata.get("credential_id"),
-                "dify.dataset.ids": self._maybe_json(metadata.get("dataset_ids")),
-                "dify.dataset.names": self._maybe_json(metadata.get("dataset_names")),
-            }
-        )
-
-        ref = f"ref:node_execution_id={info.node_execution_id}"
-        log_attrs["dify.node.inputs"] = self._content_or_ref(info.node_inputs, ref)
-        log_attrs["dify.node.outputs"] = self._content_or_ref(info.node_outputs, ref)
-        log_attrs["dify.node.process_data"] = self._content_or_ref(info.process_data, ref)
-
-        emit_telemetry_log(
-            event_name=span_name.value,
-            attributes=log_attrs,
-            signal="span_detail",
-            trace_id_source=info.workflow_run_id,
-            span_id_source=info.node_execution_id,
-            tenant_id=tenant_id,
-            user_id=user_id,
-        )
-
-        # -- Metrics --
-        labels = self._labels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-            node_type=info.node_type,
-            model_provider=info.model_provider or "",
-        )
-        if info.total_tokens:
-            token_labels = TokenMetricLabels(
-                tenant_id=tenant_id or "",
-                app_id=app_id or "",
-                operation_type=OperationType.NODE_EXECUTION,
-                model_provider=info.model_provider or "",
-                model_name=info.model_name or "",
-                node_type=info.node_type,
-            ).to_dict()
-            self._exporter.increment_counter(EnterpriseTelemetryCounter.TOKENS, info.total_tokens, token_labels)
-            if info.prompt_tokens is not None and info.prompt_tokens > 0:
-                self._exporter.increment_counter(
-                    EnterpriseTelemetryCounter.INPUT_TOKENS, info.prompt_tokens, token_labels
-                )
-            if info.completion_tokens is not None and info.completion_tokens > 0:
-                self._exporter.increment_counter(
-                    EnterpriseTelemetryCounter.OUTPUT_TOKENS, info.completion_tokens, token_labels
-                )
-        self._exporter.increment_counter(
-            EnterpriseTelemetryCounter.REQUESTS,
-            1,
-            self._labels(
-                **labels,
-                type=request_type,
-                status=info.status,
-                model_name=info.model_name or "",
-            ),
-        )
-        duration_labels = dict(labels)
-        duration_labels["model_name"] = info.model_name or ""
-        plugin_name = metadata.get("plugin_name")
-        if plugin_name and info.node_type in {"tool", "knowledge-retrieval"}:
-            duration_labels["plugin_name"] = plugin_name
-        self._exporter.record_histogram(EnterpriseTelemetryHistogram.NODE_DURATION, info.elapsed_time, duration_labels)
-
-        if info.error:
-            self._exporter.increment_counter(
-                EnterpriseTelemetryCounter.ERRORS,
-                1,
-                self._labels(
-                    **labels,
-                    type=request_type,
-                    model_name=info.model_name or "",
-                ),
-            )
-
-    # ------------------------------------------------------------------
-    # METRIC-ONLY handlers (structured log + counters/histograms)
-    # ------------------------------------------------------------------
-
-    def _message_trace(self, info: MessageTraceInfo) -> None:
-        metadata = self._metadata(info)
-        tenant_id, app_id, user_id = self._context_ids(info, metadata)
-        attrs = self._common_attrs(info)
-        attrs.update(
-            {
-                "dify.invoke_from": metadata.get("from_source"),
-                "dify.conversation.id": metadata.get("conversation_id"),
-                "dify.conversation.mode": info.conversation_mode,
-                "gen_ai.provider.name": metadata.get("ls_provider"),
-                "gen_ai.request.model": metadata.get("ls_model_name"),
-                "gen_ai.usage.input_tokens": info.message_tokens,
-                "gen_ai.usage.output_tokens": info.answer_tokens,
-                "gen_ai.usage.total_tokens": info.total_tokens,
-                "dify.message.status": metadata.get("status"),
-                "dify.message.error": info.error,
-                "dify.message.from_source": metadata.get("from_source"),
-                "dify.message.from_end_user_id": metadata.get("from_end_user_id"),
-                "dify.message.from_account_id": metadata.get("from_account_id"),
-                "dify.streaming": info.is_streaming_request,
-                "dify.message.time_to_first_token": info.gen_ai_server_time_to_first_token,
-                "dify.message.streaming_duration": info.llm_streaming_time_to_generate,
-                "dify.workflow.run_id": metadata.get("workflow_run_id"),
-            }
-        )
-        node_execution_id = metadata.get("node_execution_id")
-        if node_execution_id:
-            attrs["dify.node.execution_id"] = node_execution_id
-
-        ref = f"ref:message_id={info.message_id}"
-        inputs = self._safe_payload_value(info.inputs)
-        outputs = self._safe_payload_value(info.outputs)
-        attrs["dify.message.inputs"] = self._content_or_ref(inputs, ref)
-        attrs["dify.message.outputs"] = self._content_or_ref(outputs, ref)
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.MESSAGE_RUN,
-            attributes=attrs,
-            trace_id_source=metadata.get("workflow_run_id") or str(info.message_id) if info.message_id else None,
-            span_id_source=node_execution_id,
-            tenant_id=tenant_id,
-            user_id=user_id,
-        )
-
-        labels = self._labels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-            model_provider=metadata.get("ls_provider") or "",
-            model_name=metadata.get("ls_model_name") or "",
-        )
-        token_labels = TokenMetricLabels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-            operation_type=OperationType.MESSAGE,
-            model_provider=metadata.get("ls_provider") or "",
-            model_name=metadata.get("ls_model_name") or "",
-            node_type="",
-        ).to_dict()
-        self._exporter.increment_counter(EnterpriseTelemetryCounter.TOKENS, info.total_tokens, token_labels)
-        if info.message_tokens > 0:
-            self._exporter.increment_counter(EnterpriseTelemetryCounter.INPUT_TOKENS, info.message_tokens, token_labels)
-        if info.answer_tokens > 0:
-            self._exporter.increment_counter(EnterpriseTelemetryCounter.OUTPUT_TOKENS, info.answer_tokens, token_labels)
-        invoke_from = metadata.get("from_source", "")
-        self._exporter.increment_counter(
-            EnterpriseTelemetryCounter.REQUESTS,
-            1,
-            self._labels(
-                **labels,
-                type="message",
-                status=metadata.get("status", ""),
-                invoke_from=invoke_from,
-            ),
-        )
-
-        if info.start_time and info.end_time:
-            duration = (info.end_time - info.start_time).total_seconds()
-            self._exporter.record_histogram(EnterpriseTelemetryHistogram.MESSAGE_DURATION, duration, labels)
-
-        if info.gen_ai_server_time_to_first_token is not None:
-            self._exporter.record_histogram(
-                EnterpriseTelemetryHistogram.MESSAGE_TTFT, info.gen_ai_server_time_to_first_token, labels
-            )
-
-        if info.error:
-            self._exporter.increment_counter(
-                EnterpriseTelemetryCounter.ERRORS,
-                1,
-                self._labels(
-                    **labels,
-                    type="message",
-                ),
-            )
-
-    def _tool_trace(self, info: ToolTraceInfo) -> None:
-        metadata = self._metadata(info)
-        tenant_id, app_id, user_id = self._context_ids(info, metadata)
-        attrs = self._common_attrs(info)
-        attrs.update(
-            {
-                "gen_ai.tool.name": info.tool_name,
-                "dify.tool.time_cost": info.time_cost,
-                "dify.tool.error": info.error,
-                "dify.workflow.run_id": metadata.get("workflow_run_id"),
-            }
-        )
-        node_execution_id = metadata.get("node_execution_id")
-        if node_execution_id:
-            attrs["dify.node.execution_id"] = node_execution_id
-
-        ref = f"ref:message_id={info.message_id}"
-        attrs["dify.tool.inputs"] = self._content_or_ref(info.tool_inputs, ref)
-        attrs["dify.tool.outputs"] = self._content_or_ref(info.tool_outputs, ref)
-        attrs["dify.tool.parameters"] = self._content_or_ref(info.tool_parameters, ref)
-        attrs["dify.tool.config"] = self._content_or_ref(info.tool_config, ref)
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.TOOL_EXECUTION,
-            attributes=attrs,
-            trace_id_source=info.resolved_trace_id,
-            span_id_source=node_execution_id,
-            tenant_id=tenant_id,
-            user_id=user_id,
-        )
-
-        labels = self._labels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-            tool_name=info.tool_name,
-        )
-        self._exporter.increment_counter(
-            EnterpriseTelemetryCounter.REQUESTS,
-            1,
-            self._labels(
-                **labels,
-                type="tool",
-            ),
-        )
-        self._exporter.record_histogram(EnterpriseTelemetryHistogram.TOOL_DURATION, float(info.time_cost), labels)
-
-        if info.error:
-            self._exporter.increment_counter(
-                EnterpriseTelemetryCounter.ERRORS,
-                1,
-                self._labels(
-                    **labels,
-                    type="tool",
-                ),
-            )
-
-    def _moderation_trace(self, info: ModerationTraceInfo) -> None:
-        metadata = self._metadata(info)
-        tenant_id, app_id, user_id = self._context_ids(info, metadata)
-        attrs = self._common_attrs(info)
-        attrs.update(
-            {
-                "dify.moderation.flagged": info.flagged,
-                "dify.moderation.action": info.action,
-                "dify.moderation.preset_response": info.preset_response,
-                "dify.workflow.run_id": metadata.get("workflow_run_id"),
-            }
-        )
-        node_execution_id = metadata.get("node_execution_id")
-        if node_execution_id:
-            attrs["dify.node.execution_id"] = node_execution_id
-
-        attrs["dify.moderation.query"] = self._content_or_ref(
-            info.query,
-            f"ref:message_id={info.message_id}",
-        )
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.MODERATION_CHECK,
-            attributes=attrs,
-            trace_id_source=info.resolved_trace_id,
-            span_id_source=node_execution_id,
-            tenant_id=tenant_id,
-            user_id=user_id,
-        )
-
-        labels = self._labels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-        )
-        self._exporter.increment_counter(
-            EnterpriseTelemetryCounter.REQUESTS,
-            1,
-            self._labels(
-                **labels,
-                type="moderation",
-            ),
-        )
-
-    def _suggested_question_trace(self, info: SuggestedQuestionTraceInfo) -> None:
-        metadata = self._metadata(info)
-        tenant_id, app_id, user_id = self._context_ids(info, metadata)
-        attrs = self._common_attrs(info)
-        attrs.update(
-            {
-                "gen_ai.usage.total_tokens": info.total_tokens,
-                "dify.suggested_question.status": info.status,
-                "dify.suggested_question.error": info.error,
-                "gen_ai.provider.name": info.model_provider,
-                "gen_ai.request.model": info.model_id,
-                "dify.suggested_question.count": len(info.suggested_question),
-                "dify.workflow.run_id": metadata.get("workflow_run_id"),
-            }
-        )
-        node_execution_id = metadata.get("node_execution_id")
-        if node_execution_id:
-            attrs["dify.node.execution_id"] = node_execution_id
-
-        attrs["dify.suggested_question.questions"] = self._content_or_ref(
-            info.suggested_question,
-            f"ref:message_id={info.message_id}",
-        )
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.SUGGESTED_QUESTION_GENERATION,
-            attributes=attrs,
-            trace_id_source=info.resolved_trace_id,
-            span_id_source=node_execution_id,
-            tenant_id=tenant_id,
-            user_id=user_id,
-        )
-
-        labels = self._labels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-        )
-        self._exporter.increment_counter(
-            EnterpriseTelemetryCounter.REQUESTS,
-            1,
-            self._labels(
-                **labels,
-                type="suggested_question",
-                model_provider=info.model_provider or "",
-                model_name=info.model_id or "",
-            ),
-        )
-
-    def _dataset_retrieval_trace(self, info: DatasetRetrievalTraceInfo) -> None:
-        metadata = self._metadata(info)
-        tenant_id, app_id, user_id = self._context_ids(info, metadata)
-        attrs = self._common_attrs(info)
-        attrs["dify.dataset.error"] = info.error
-        attrs["dify.workflow.run_id"] = metadata.get("workflow_run_id")
-        node_execution_id = metadata.get("node_execution_id")
-        if node_execution_id:
-            attrs["dify.node.execution_id"] = node_execution_id
-
-        docs: list[dict[str, Any]] = []
-        documents_any: Any = info.documents
-        documents_list: list[Any] = cast(list[Any], documents_any) if isinstance(documents_any, list) else []
-        for entry in documents_list:
-            if isinstance(entry, dict):
-                entry_dict: dict[str, Any] = cast(dict[str, Any], entry)
-                docs.append(entry_dict)
-        dataset_ids: list[str] = []
-        dataset_names: list[str] = []
-        structured_docs: list[dict[str, Any]] = []
-        for doc in docs:
-            meta_raw = doc.get("metadata")
-            meta: dict[str, Any] = cast(dict[str, Any], meta_raw) if isinstance(meta_raw, dict) else {}
-            did = meta.get("dataset_id")
-            dname = meta.get("dataset_name")
-            if did and did not in dataset_ids:
-                dataset_ids.append(did)
-            if dname and dname not in dataset_names:
-                dataset_names.append(dname)
-            structured_docs.append(
-                {
-                    "dataset_id": did,
-                    "document_id": meta.get("document_id"),
-                    "segment_id": meta.get("segment_id"),
-                    "score": meta.get("score"),
-                }
-            )
-
-        attrs["dify.dataset.ids"] = self._maybe_json(dataset_ids)
-        attrs["dify.dataset.names"] = self._maybe_json(dataset_names)
-        attrs["dify.retrieval.document_count"] = len(docs)
-
-        embedding_models_raw: Any = metadata.get("embedding_models")
-        embedding_models: dict[str, Any] = (
-            cast(dict[str, Any], embedding_models_raw) if isinstance(embedding_models_raw, dict) else {}
-        )
-        if embedding_models:
-            providers: list[str] = []
-            models: list[str] = []
-            for ds_info in embedding_models.values():
-                if isinstance(ds_info, dict):
-                    ds_info_dict: dict[str, Any] = cast(dict[str, Any], ds_info)
-                    p = ds_info_dict.get("embedding_model_provider", "")
-                    m = ds_info_dict.get("embedding_model", "")
-                    if p and p not in providers:
-                        providers.append(p)
-                    if m and m not in models:
-                        models.append(m)
-            attrs["dify.dataset.embedding_providers"] = self._maybe_json(providers)
-            attrs["dify.dataset.embedding_models"] = self._maybe_json(models)
-
-        # Add rerank model to logs
-        rerank_provider = metadata.get("rerank_model_provider", "")
-        rerank_model = metadata.get("rerank_model_name", "")
-        if rerank_provider or rerank_model:
-            attrs["dify.retrieval.rerank_provider"] = rerank_provider
-            attrs["dify.retrieval.rerank_model"] = rerank_model
-
-        ref = f"ref:message_id={info.message_id}"
-        retrieval_inputs = self._safe_payload_value(info.inputs)
-        attrs["dify.retrieval.query"] = self._content_or_ref(retrieval_inputs, ref)
-        attrs["dify.dataset.documents"] = self._content_or_ref(structured_docs, ref)
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.DATASET_RETRIEVAL,
-            attributes=attrs,
-            trace_id_source=metadata.get("workflow_run_id") or str(info.message_id) if info.message_id else None,
-            span_id_source=node_execution_id or (str(info.message_id) if info.message_id else None),
-            tenant_id=tenant_id,
-            user_id=user_id,
-        )
-
-        labels = self._labels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-        )
-        self._exporter.increment_counter(
-            EnterpriseTelemetryCounter.REQUESTS,
-            1,
-            self._labels(
-                **labels,
-                type="dataset_retrieval",
-            ),
-        )
-
-        for did in dataset_ids:
-            # Get embedding model for this specific dataset
-            ds_embedding_info = embedding_models.get(did, {})
-            embedding_provider = ds_embedding_info.get("embedding_model_provider", "")
-            embedding_model = ds_embedding_info.get("embedding_model", "")
-
-            # Get rerank model (same for all datasets in this retrieval)
-            rerank_provider = metadata.get("rerank_model_provider", "")
-            rerank_model = metadata.get("rerank_model_name", "")
-
-            self._exporter.increment_counter(
-                EnterpriseTelemetryCounter.DATASET_RETRIEVALS,
-                1,
-                self._labels(
-                    **labels,
-                    dataset_id=did,
-                    embedding_model_provider=embedding_provider,
-                    embedding_model=embedding_model,
-                    rerank_model_provider=rerank_provider,
-                    rerank_model=rerank_model,
-                ),
-            )
-
-    def _generate_name_trace(self, info: GenerateNameTraceInfo) -> None:
-        metadata = self._metadata(info)
-        tenant_id, app_id, user_id = self._context_ids(info, metadata)
-        attrs = self._common_attrs(info)
-        attrs["dify.conversation.id"] = info.conversation_id
-        node_execution_id = metadata.get("node_execution_id")
-        if node_execution_id:
-            attrs["dify.node.execution_id"] = node_execution_id
-
-        ref = f"ref:conversation_id={info.conversation_id}"
-        inputs = self._safe_payload_value(info.inputs)
-        outputs = self._safe_payload_value(info.outputs)
-        attrs["dify.generate_name.inputs"] = self._content_or_ref(inputs, ref)
-        attrs["dify.generate_name.outputs"] = self._content_or_ref(outputs, ref)
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.GENERATE_NAME_EXECUTION,
-            attributes=attrs,
-            trace_id_source=info.resolved_trace_id,
-            span_id_source=node_execution_id,
-            tenant_id=tenant_id,
-            user_id=user_id,
-        )
-
-        labels = self._labels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-        )
-        self._exporter.increment_counter(
-            EnterpriseTelemetryCounter.REQUESTS,
-            1,
-            self._labels(
-                **labels,
-                type="generate_name",
-            ),
-        )
-
-    def _prompt_generation_trace(self, info: PromptGenerationTraceInfo) -> None:
-        metadata = self._metadata(info)
-        tenant_id, app_id, user_id = self._context_ids(info, metadata)
-        attrs = {
-            "dify.trace_id": info.resolved_trace_id,
-            "dify.tenant_id": tenant_id,
-            "dify.user.id": user_id,
-            "dify.app.id": app_id or "",
-            "dify.app.name": metadata.get("app_name"),
-            "dify.workspace.name": metadata.get("workspace_name"),
-            "dify.operation.type": info.operation_type,
-            "gen_ai.provider.name": info.model_provider,
-            "gen_ai.request.model": info.model_name,
-            "gen_ai.usage.input_tokens": info.prompt_tokens,
-            "gen_ai.usage.output_tokens": info.completion_tokens,
-            "gen_ai.usage.total_tokens": info.total_tokens,
-            "dify.prompt_generation.latency": info.latency,
-            "dify.prompt_generation.error": info.error,
-        }
-        node_execution_id = metadata.get("node_execution_id")
-        if node_execution_id:
-            attrs["dify.node.execution_id"] = node_execution_id
-
-        if info.total_price is not None:
-            attrs["dify.prompt_generation.total_price"] = info.total_price
-            attrs["dify.prompt_generation.currency"] = info.currency
-
-        ref = f"ref:trace_id={info.trace_id}"
-        outputs = self._safe_payload_value(info.outputs)
-        attrs["dify.prompt_generation.instruction"] = self._content_or_ref(info.instruction, ref)
-        attrs["dify.prompt_generation.output"] = self._content_or_ref(outputs, ref)
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.PROMPT_GENERATION_EXECUTION,
-            attributes=attrs,
-            trace_id_source=info.resolved_trace_id,
-            span_id_source=node_execution_id,
-            tenant_id=tenant_id,
-            user_id=user_id,
-        )
-
-        token_labels = TokenMetricLabels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-            operation_type=info.operation_type,
-            model_provider=info.model_provider,
-            model_name=info.model_name,
-            node_type="",
-        ).to_dict()
-
-        labels = self._labels(
-            tenant_id=tenant_id or "",
-            app_id=app_id or "",
-            operation_type=info.operation_type,
-            model_provider=info.model_provider,
-            model_name=info.model_name,
-        )
-
-        self._exporter.increment_counter(EnterpriseTelemetryCounter.TOKENS, info.total_tokens, token_labels)
-        if info.prompt_tokens > 0:
-            self._exporter.increment_counter(EnterpriseTelemetryCounter.INPUT_TOKENS, info.prompt_tokens, token_labels)
-        if info.completion_tokens > 0:
-            self._exporter.increment_counter(
-                EnterpriseTelemetryCounter.OUTPUT_TOKENS, info.completion_tokens, token_labels
-            )
-
-        status = "failed" if info.error else "success"
-        self._exporter.increment_counter(
-            EnterpriseTelemetryCounter.REQUESTS,
-            1,
-            self._labels(
-                **labels,
-                type="prompt_generation",
-                status=status,
-            ),
-        )
-
-        self._exporter.record_histogram(
-            EnterpriseTelemetryHistogram.PROMPT_GENERATION_DURATION,
-            info.latency,
-            labels,
-        )
-
-        if info.error:
-            self._exporter.increment_counter(
-                EnterpriseTelemetryCounter.ERRORS,
-                1,
-                self._labels(
-                    **labels,
-                    type="prompt_generation",
-                ),
-            )

api/enterprise/telemetry/entities/__init__.py

@@ -1,121 +0,0 @@
-from enum import StrEnum
-from typing import cast
-
-from opentelemetry.util.types import AttributeValue
-from pydantic import BaseModel, ConfigDict
-
-
-class EnterpriseTelemetrySpan(StrEnum):
-    WORKFLOW_RUN = "dify.workflow.run"
-    NODE_EXECUTION = "dify.node.execution"
-    DRAFT_NODE_EXECUTION = "dify.node.execution.draft"
-
-
-class EnterpriseTelemetryEvent(StrEnum):
-    """Event names for enterprise telemetry logs."""
-
-    APP_CREATED = "dify.app.created"
-    APP_UPDATED = "dify.app.updated"
-    APP_DELETED = "dify.app.deleted"
-    FEEDBACK_CREATED = "dify.feedback.created"
-    WORKFLOW_RUN = "dify.workflow.run"
-    MESSAGE_RUN = "dify.message.run"
-    TOOL_EXECUTION = "dify.tool.execution"
-    MODERATION_CHECK = "dify.moderation.check"
-    SUGGESTED_QUESTION_GENERATION = "dify.suggested_question.generation"
-    DATASET_RETRIEVAL = "dify.dataset.retrieval"
-    GENERATE_NAME_EXECUTION = "dify.generate_name.execution"
-    PROMPT_GENERATION_EXECUTION = "dify.prompt_generation.execution"
-    REHYDRATION_FAILED = "dify.telemetry.rehydration_failed"
-
-
-class EnterpriseTelemetryCounter(StrEnum):
-    TOKENS = "tokens"
-    INPUT_TOKENS = "input_tokens"
-    OUTPUT_TOKENS = "output_tokens"
-    REQUESTS = "requests"
-    ERRORS = "errors"
-    FEEDBACK = "feedback"
-    DATASET_RETRIEVALS = "dataset_retrievals"
-    APP_CREATED = "app_created"
-    APP_UPDATED = "app_updated"
-    APP_DELETED = "app_deleted"
-
-
-class EnterpriseTelemetryHistogram(StrEnum):
-    WORKFLOW_DURATION = "workflow_duration"
-    NODE_DURATION = "node_duration"
-    MESSAGE_DURATION = "message_duration"
-    MESSAGE_TTFT = "message_ttft"
-    TOOL_DURATION = "tool_duration"
-    PROMPT_GENERATION_DURATION = "prompt_generation_duration"
-
-
-class TokenMetricLabels(BaseModel):
-    """Unified label structure for all dify.token.* metrics.
-
-    All token counters (dify.tokens.input, dify.tokens.output, dify.tokens.total) MUST
-    use this exact label set to ensure consistent filtering and aggregation across
-    different operation types.
-
-    Attributes:
-        tenant_id: Tenant identifier.
-        app_id: Application identifier.
-        operation_type: Source of token usage (workflow | node_execution | message |
-            rule_generate | code_generate | structured_output | instruction_modify).
-        model_provider: LLM provider name. Empty string if not applicable (e.g., workflow-level).
-        model_name: LLM model name. Empty string if not applicable (e.g., workflow-level).
-        node_type: Workflow node type. Empty string unless operation_type=node_execution.
-
-    Usage:
-        labels = TokenMetricLabels(
-            tenant_id="tenant-123",
-            app_id="app-456",
-            operation_type=OperationType.WORKFLOW,
-            model_provider="",
-            model_name="",
-            node_type="",
-        )
-        exporter.increment_counter(
-            EnterpriseTelemetryCounter.INPUT_TOKENS,
-            100,
-            labels.to_dict()
-        )
-
-    Design rationale:
-        Without this unified structure, tokens get double-counted when querying totals
-        because workflow.total_tokens is already the sum of all node tokens. The
-        operation_type label allows filtering to separate workflow-level aggregates from
-        node-level detail, while keeping the same label cardinality for consistent queries.
-    """
-
-    tenant_id: str
-    app_id: str
-    operation_type: str
-    model_provider: str
-    model_name: str
-    node_type: str
-
-    model_config = ConfigDict(extra="forbid", frozen=True)
-
-    def to_dict(self) -> dict[str, AttributeValue]:
-        return cast(
-            dict[str, AttributeValue],
-            {
-                "tenant_id": self.tenant_id,
-                "app_id": self.app_id,
-                "operation_type": self.operation_type,
-                "model_provider": self.model_provider,
-                "model_name": self.model_name,
-                "node_type": self.node_type,
-            },
-        )
-
-
-__all__ = [
-    "EnterpriseTelemetryCounter",
-    "EnterpriseTelemetryEvent",
-    "EnterpriseTelemetryHistogram",
-    "EnterpriseTelemetrySpan",
-    "TokenMetricLabels",
-]

api/enterprise/telemetry/event_handlers.py

@@ -1,99 +0,0 @@
-"""Blinker signal handlers for enterprise telemetry.
-
-Registered at import time via ``@signal.connect`` decorators.
-Import must happen during ``ext_enterprise_telemetry.init_app()`` to
-ensure handlers fire.  Each handler delegates to ``core.telemetry.gateway``
-which handles routing, EE-gating, and dispatch.
-
-All handlers are best-effort: exceptions are caught and logged so that
-telemetry failures never break user-facing operations.
-"""
-
-from __future__ import annotations
-
-import logging
-
-from events.app_event import app_was_created, app_was_deleted, app_was_updated
-from events.feedback_event import feedback_was_created
-
-logger = logging.getLogger(__name__)
-
-__all__ = [
-    "_handle_app_created",
-    "_handle_app_deleted",
-    "_handle_app_updated",
-    "_handle_feedback_created",
-]
-
-
-@app_was_created.connect
-def _handle_app_created(sender: object, **kwargs: object) -> None:
-    try:
-        from core.telemetry.gateway import emit as gateway_emit
-        from enterprise.telemetry.contracts import TelemetryCase
-
-        gateway_emit(
-            case=TelemetryCase.APP_CREATED,
-            context={"tenant_id": str(getattr(sender, "tenant_id", "") or "")},
-            payload={
-                "app_id": getattr(sender, "id", None),
-                "mode": getattr(sender, "mode", None),
-            },
-        )
-    except Exception:
-        logger.warning("Failed to emit app_created telemetry", exc_info=True)
-
-
-@app_was_deleted.connect
-def _handle_app_deleted(sender: object, **kwargs: object) -> None:
-    try:
-        from core.telemetry.gateway import emit as gateway_emit
-        from enterprise.telemetry.contracts import TelemetryCase
-
-        gateway_emit(
-            case=TelemetryCase.APP_DELETED,
-            context={"tenant_id": str(getattr(sender, "tenant_id", "") or "")},
-            payload={"app_id": getattr(sender, "id", None)},
-        )
-    except Exception:
-        logger.warning("Failed to emit app_deleted telemetry", exc_info=True)
-
-
-@app_was_updated.connect
-def _handle_app_updated(sender: object, **kwargs: object) -> None:
-    try:
-        from core.telemetry.gateway import emit as gateway_emit
-        from enterprise.telemetry.contracts import TelemetryCase
-
-        gateway_emit(
-            case=TelemetryCase.APP_UPDATED,
-            context={"tenant_id": str(getattr(sender, "tenant_id", "") or "")},
-            payload={"app_id": getattr(sender, "id", None)},
-        )
-    except Exception:
-        logger.warning("Failed to emit app_updated telemetry", exc_info=True)
-
-
-@feedback_was_created.connect
-def _handle_feedback_created(sender: object, **kwargs: object) -> None:
-    try:
-        from core.telemetry.gateway import emit as gateway_emit
-        from enterprise.telemetry.contracts import TelemetryCase
-
-        tenant_id = str(kwargs.get("tenant_id", "") or "")
-        gateway_emit(
-            case=TelemetryCase.FEEDBACK_CREATED,
-            context={"tenant_id": tenant_id},
-            payload={
-                "message_id": getattr(sender, "message_id", None),
-                "app_id": getattr(sender, "app_id", None),
-                "conversation_id": getattr(sender, "conversation_id", None),
-                "from_end_user_id": getattr(sender, "from_end_user_id", None),
-                "from_account_id": getattr(sender, "from_account_id", None),
-                "rating": getattr(sender, "rating", None),
-                "from_source": getattr(sender, "from_source", None),
-                "content": getattr(sender, "content", None),
-            },
-        )
-    except Exception:
-        logger.warning("Failed to emit feedback_created telemetry", exc_info=True)

api/enterprise/telemetry/exporter.py

@@ -1,284 +0,0 @@
-"""Enterprise OTEL exporter — shared by EnterpriseOtelTrace, event handlers, and direct instrumentation.
-
-Uses dedicated TracerProvider and MeterProvider instances (configurable sampling,
-independent from ext_otel.py infrastructure).
-
-Initialized once during Flask extension init (single-threaded via ext_enterprise_telemetry.py).
-Accessed via ``ext_enterprise_telemetry.get_enterprise_exporter()`` from any thread/process.
-"""
-
-import logging
-import socket
-import uuid
-from datetime import datetime
-from typing import Any, cast
-
-from opentelemetry import trace
-from opentelemetry.context import Context
-from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter as GRPCMetricExporter
-from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter as GRPCSpanExporter
-from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter as HTTPMetricExporter
-from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as HTTPSpanExporter
-from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
-from opentelemetry.sdk.resources import Resource
-from opentelemetry.sdk.trace import TracerProvider
-from opentelemetry.sdk.trace.export import BatchSpanProcessor
-from opentelemetry.sdk.trace.sampling import ParentBasedTraceIdRatio
-from opentelemetry.semconv.resource import ResourceAttributes
-from opentelemetry.trace import SpanContext, TraceFlags
-from opentelemetry.util.types import Attributes, AttributeValue
-
-from configs import dify_config
-from enterprise.telemetry.entities import EnterpriseTelemetryCounter, EnterpriseTelemetryHistogram
-from enterprise.telemetry.id_generator import (
-    CorrelationIdGenerator,
-    compute_deterministic_span_id,
-    set_correlation_id,
-    set_span_id_source,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def is_enterprise_telemetry_enabled() -> bool:
-    return bool(dify_config.ENTERPRISE_ENABLED and dify_config.ENTERPRISE_TELEMETRY_ENABLED)
-
-
-def _parse_otlp_headers(raw: str) -> dict[str, str]:
-    """Parse ``key=value,key2=value2`` into a dict."""
-    if not raw:
-        return {}
-    headers: dict[str, str] = {}
-    for pair in raw.split(","):
-        if "=" not in pair:
-            continue
-        k, v = pair.split("=", 1)
-        headers[k.strip()] = v.strip()
-    return headers
-
-
-def _datetime_to_ns(dt: datetime) -> int:
-    """Convert a datetime to nanoseconds since epoch (OTEL convention)."""
-    return int(dt.timestamp() * 1_000_000_000)
-
-
-class _ExporterFactory:
-    def __init__(self, protocol: str, endpoint: str, headers: dict[str, str], insecure: bool):
-        self._protocol = protocol
-        self._endpoint = endpoint
-        self._headers = headers
-        self._grpc_headers = tuple(headers.items()) if headers else None
-        self._http_headers = headers or None
-        self._insecure = insecure
-
-    def create_trace_exporter(self) -> HTTPSpanExporter | GRPCSpanExporter:
-        if self._protocol == "grpc":
-            return GRPCSpanExporter(
-                endpoint=self._endpoint or None,
-                headers=self._grpc_headers,
-                insecure=self._insecure,
-            )
-        trace_endpoint = f"{self._endpoint}/v1/traces" if self._endpoint else ""
-        return HTTPSpanExporter(endpoint=trace_endpoint or None, headers=self._http_headers)
-
-    def create_metric_exporter(self) -> HTTPMetricExporter | GRPCMetricExporter:
-        if self._protocol == "grpc":
-            return GRPCMetricExporter(
-                endpoint=self._endpoint or None,
-                headers=self._grpc_headers,
-                insecure=self._insecure,
-            )
-        metric_endpoint = f"{self._endpoint}/v1/metrics" if self._endpoint else ""
-        return HTTPMetricExporter(endpoint=metric_endpoint or None, headers=self._http_headers)
-
-
-class EnterpriseExporter:
-    """Shared OTEL exporter for all enterprise telemetry.
-
-    ``export_span`` creates spans with optional real timestamps, deterministic
-    span/trace IDs, and cross-workflow parent linking.
-    ``increment_counter`` / ``record_histogram`` emit OTEL metrics at 100% accuracy.
-    """
-
-    def __init__(self, config: object) -> None:
-        endpoint: str = getattr(config, "ENTERPRISE_OTLP_ENDPOINT", "")
-        headers_raw: str = getattr(config, "ENTERPRISE_OTLP_HEADERS", "")
-        protocol: str = (getattr(config, "ENTERPRISE_OTLP_PROTOCOL", "http") or "http").lower()
-        service_name: str = getattr(config, "ENTERPRISE_SERVICE_NAME", "dify")
-        sampling_rate: float = getattr(config, "ENTERPRISE_OTEL_SAMPLING_RATE", 1.0)
-        self.include_content: bool = getattr(config, "ENTERPRISE_INCLUDE_CONTENT", True)
-        api_key: str = getattr(config, "ENTERPRISE_OTLP_API_KEY", "")
-
-        # Auto-detect TLS: https:// uses secure, everything else is insecure
-        insecure = not endpoint.startswith("https://")
-
-        resource = Resource(
-            attributes={
-                ResourceAttributes.SERVICE_NAME: service_name,
-                ResourceAttributes.HOST_NAME: socket.gethostname(),
-            }
-        )
-        sampler = ParentBasedTraceIdRatio(sampling_rate)
-        id_generator = CorrelationIdGenerator()
-        self._tracer_provider = TracerProvider(resource=resource, sampler=sampler, id_generator=id_generator)
-
-        headers = _parse_otlp_headers(headers_raw)
-        if api_key:
-            if "authorization" in headers:
-                logger.warning(
-                    "ENTERPRISE_OTLP_API_KEY is set but ENTERPRISE_OTLP_HEADERS also contains "
-                    "'authorization'; the API key will take precedence."
-                )
-            headers["authorization"] = f"Bearer {api_key}"
-        factory = _ExporterFactory(protocol, endpoint, headers, insecure=insecure)
-
-        trace_exporter = factory.create_trace_exporter()
-        self._tracer_provider.add_span_processor(BatchSpanProcessor(trace_exporter))
-        self._tracer = self._tracer_provider.get_tracer("dify.enterprise")
-
-        metric_exporter = factory.create_metric_exporter()
-        self._meter_provider = MeterProvider(
-            resource=resource,
-            metric_readers=[PeriodicExportingMetricReader(metric_exporter)],
-        )
-        meter = self._meter_provider.get_meter("dify.enterprise")
-        self._counters = {
-            EnterpriseTelemetryCounter.TOKENS: meter.create_counter("dify.tokens.total", unit="{token}"),
-            EnterpriseTelemetryCounter.INPUT_TOKENS: meter.create_counter("dify.tokens.input", unit="{token}"),
-            EnterpriseTelemetryCounter.OUTPUT_TOKENS: meter.create_counter("dify.tokens.output", unit="{token}"),
-            EnterpriseTelemetryCounter.REQUESTS: meter.create_counter("dify.requests.total", unit="{request}"),
-            EnterpriseTelemetryCounter.ERRORS: meter.create_counter("dify.errors.total", unit="{error}"),
-            EnterpriseTelemetryCounter.FEEDBACK: meter.create_counter("dify.feedback.total", unit="{feedback}"),
-            EnterpriseTelemetryCounter.DATASET_RETRIEVALS: meter.create_counter(
-                "dify.dataset.retrievals.total", unit="{retrieval}"
-            ),
-            EnterpriseTelemetryCounter.APP_CREATED: meter.create_counter("dify.app.created.total", unit="{app}"),
-            EnterpriseTelemetryCounter.APP_UPDATED: meter.create_counter("dify.app.updated.total", unit="{app}"),
-            EnterpriseTelemetryCounter.APP_DELETED: meter.create_counter("dify.app.deleted.total", unit="{app}"),
-        }
-        self._histograms = {
-            EnterpriseTelemetryHistogram.WORKFLOW_DURATION: meter.create_histogram("dify.workflow.duration", unit="s"),
-            EnterpriseTelemetryHistogram.NODE_DURATION: meter.create_histogram("dify.node.duration", unit="s"),
-            EnterpriseTelemetryHistogram.MESSAGE_DURATION: meter.create_histogram("dify.message.duration", unit="s"),
-            EnterpriseTelemetryHistogram.MESSAGE_TTFT: meter.create_histogram(
-                "dify.message.time_to_first_token", unit="s"
-            ),
-            EnterpriseTelemetryHistogram.TOOL_DURATION: meter.create_histogram("dify.tool.duration", unit="s"),
-            EnterpriseTelemetryHistogram.PROMPT_GENERATION_DURATION: meter.create_histogram(
-                "dify.prompt_generation.duration", unit="s"
-            ),
-        }
-
-    def export_span(
-        self,
-        name: str,
-        attributes: dict[str, Any],
-        correlation_id: str | None = None,
-        span_id_source: str | None = None,
-        start_time: datetime | None = None,
-        end_time: datetime | None = None,
-        trace_correlation_override: str | None = None,
-        parent_span_id_source: str | None = None,
-    ) -> None:
-        """Export an OTEL span with optional deterministic IDs and real timestamps.
-
-        Args:
-            name: Span operation name.
-            attributes: Span attributes dict.
-            correlation_id: Source for trace_id derivation (groups spans in one trace).
-            span_id_source: Source for deterministic span_id (e.g. workflow_run_id or node_execution_id).
-            start_time: Real span start time. When None, uses current time.
-            end_time: Real span end time. When None, span ends immediately.
-            trace_correlation_override: Override trace_id source (for cross-workflow linking).
-                When set, trace_id is derived from this instead of ``correlation_id``.
-            parent_span_id_source: Override parent span_id source (for cross-workflow linking).
-                When set, parent span_id is derived from this value. When None and
-                ``correlation_id`` is set, parent is the workflow root span.
-        """
-        effective_trace_correlation = trace_correlation_override or correlation_id
-        set_correlation_id(effective_trace_correlation)
-        set_span_id_source(span_id_source)
-
-        try:
-            parent_context: Context | None = None
-            # A span is the "root" of its correlation group when span_id_source == correlation_id
-            # (i.e. a workflow root span).  All other spans are children.
-            if parent_span_id_source:
-                # Cross-workflow linking: parent is an explicit span (e.g. tool node in outer workflow)
-                parent_span_id = compute_deterministic_span_id(parent_span_id_source)
-                try:
-                    parent_trace_id = int(uuid.UUID(effective_trace_correlation)) if effective_trace_correlation else 0
-                except (ValueError, AttributeError):
-                    logger.warning(
-                        "Invalid trace correlation UUID for cross-workflow link: %s, span=%s",
-                        effective_trace_correlation,
-                        name,
-                    )
-                    parent_trace_id = 0
-                if parent_trace_id:
-                    parent_span_context = SpanContext(
-                        trace_id=parent_trace_id,
-                        span_id=parent_span_id,
-                        is_remote=True,
-                        trace_flags=TraceFlags(TraceFlags.SAMPLED),
-                    )
-                    parent_context = trace.set_span_in_context(trace.NonRecordingSpan(parent_span_context))
-            elif correlation_id and correlation_id != span_id_source:
-                # Child span: parent is the correlation-group root (workflow root span)
-                parent_span_id = compute_deterministic_span_id(correlation_id)
-                try:
-                    parent_trace_id = int(uuid.UUID(effective_trace_correlation or correlation_id))
-                except (ValueError, AttributeError):
-                    logger.warning(
-                        "Invalid trace correlation UUID for child span link: %s, span=%s",
-                        effective_trace_correlation or correlation_id,
-                        name,
-                    )
-                    parent_trace_id = 0
-                if parent_trace_id:
-                    parent_span_context = SpanContext(
-                        trace_id=parent_trace_id,
-                        span_id=parent_span_id,
-                        is_remote=True,
-                        trace_flags=TraceFlags(TraceFlags.SAMPLED),
-                    )
-                    parent_context = trace.set_span_in_context(trace.NonRecordingSpan(parent_span_context))
-
-            span_start_time = _datetime_to_ns(start_time) if start_time is not None else None
-            span_end_on_exit = end_time is None
-
-            with self._tracer.start_as_current_span(
-                name,
-                context=parent_context,
-                start_time=span_start_time,
-                end_on_exit=span_end_on_exit,
-            ) as span:
-                for key, value in attributes.items():
-                    if value is not None:
-                        span.set_attribute(key, value)
-                if end_time is not None:
-                    span.end(end_time=_datetime_to_ns(end_time))
-        except Exception:
-            logger.exception("Failed to export span %s", name)
-        finally:
-            set_correlation_id(None)
-            set_span_id_source(None)
-
-    def increment_counter(
-        self, name: EnterpriseTelemetryCounter, value: int, labels: dict[str, AttributeValue]
-    ) -> None:
-        counter = self._counters.get(name)
-        if counter:
-            counter.add(value, cast(Attributes, labels))
-
-    def record_histogram(
-        self, name: EnterpriseTelemetryHistogram, value: float, labels: dict[str, AttributeValue]
-    ) -> None:
-        histogram = self._histograms.get(name)
-        if histogram:
-            histogram.record(value, cast(Attributes, labels))
-
-    def shutdown(self) -> None:
-        self._tracer_provider.shutdown()
-        self._meter_provider.shutdown()

api/enterprise/telemetry/id_generator.py

@@ -1,76 +0,0 @@
-"""Custom OTEL ID Generator for correlation-based trace/span ID derivation.
-
-Uses contextvars for thread-safe correlation_id -> trace_id mapping.
-When a span_id_source is set, the span_id is derived deterministically
-from that value, enabling any span to reference another as parent
-without depending on span creation order.
-"""
-
-import random
-import uuid
-from contextvars import ContextVar
-from typing import cast
-
-from opentelemetry.sdk.trace.id_generator import IdGenerator
-
-_correlation_id_context: ContextVar[str | None] = ContextVar("correlation_id", default=None)
-_span_id_source_context: ContextVar[str | None] = ContextVar("span_id_source", default=None)
-
-
-def set_correlation_id(correlation_id: str | None) -> None:
-    _correlation_id_context.set(correlation_id)
-
-
-def get_correlation_id() -> str | None:
-    return _correlation_id_context.get()
-
-
-def set_span_id_source(source_id: str | None) -> None:
-    """Set the source for deterministic span_id generation.
-
-    When set, ``generate_span_id()`` derives the span_id from this value
-    (lower 64 bits of the UUID).  Pass the ``workflow_run_id`` for workflow
-    root spans or ``node_execution_id`` for node spans.
-    """
-    _span_id_source_context.set(source_id)
-
-
-def compute_deterministic_span_id(source_id: str) -> int:
-    """Derive a deterministic span_id from any UUID string.
-
-    Uses the lower 64 bits of the UUID, guaranteeing non-zero output
-    (OTEL requires span_id != 0).
-    """
-    span_id = cast(int, uuid.UUID(source_id).int) & ((1 << 64) - 1)
-    return span_id if span_id != 0 else 1
-
-
-class CorrelationIdGenerator(IdGenerator):
-    """ID generator that derives trace_id and optionally span_id from context.
-
-    - trace_id: always derived from correlation_id (groups all spans in one trace)
-    - span_id: derived from span_id_source when set (enables deterministic
-      parent-child linking), otherwise random
-    """
-
-    def generate_trace_id(self) -> int:
-        correlation_id = _correlation_id_context.get()
-        if correlation_id:
-            try:
-                return cast(int, uuid.UUID(correlation_id).int)
-            except (ValueError, AttributeError):
-                pass
-        return random.getrandbits(128)
-
-    def generate_span_id(self) -> int:
-        source = _span_id_source_context.get()
-        if source:
-            try:
-                return compute_deterministic_span_id(source)
-            except (ValueError, AttributeError):
-                pass
-
-        span_id = random.getrandbits(64)
-        while span_id == 0:
-            span_id = random.getrandbits(64)
-        return span_id

api/enterprise/telemetry/metric_handler.py

@@ -1,381 +0,0 @@
-"""Enterprise metric/log event handler.
-
-This module processes metric and log telemetry events after they've been
-dequeued from the enterprise_telemetry Celery queue. It handles case routing,
-idempotency checking, and payload rehydration.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-from typing import Any
-
-from enterprise.telemetry.contracts import TelemetryCase, TelemetryEnvelope
-from extensions.ext_redis import redis_client
-from extensions.ext_storage import storage
-
-logger = logging.getLogger(__name__)
-
-
-class EnterpriseMetricHandler:
-    """Handler for enterprise metric and log telemetry events.
-
-    Processes envelopes from the enterprise_telemetry queue, routing each
-    case to the appropriate handler method. Implements idempotency checking
-    and payload rehydration with fallback.
-    """
-
-    def _increment_diagnostic_counter(self, counter_name: str, labels: dict[str, str] | None = None) -> None:
-        """Increment a diagnostic counter for operational monitoring.
-
-        Args:
-            counter_name: Name of the counter (e.g., 'processed_total', 'deduped_total').
-            labels: Optional labels for the counter.
-        """
-        try:
-            from extensions.ext_enterprise_telemetry import get_enterprise_exporter
-
-            exporter = get_enterprise_exporter()
-            if not exporter:
-                return
-
-            full_counter_name = f"enterprise_telemetry.handler.{counter_name}"
-            logger.debug(
-                "Diagnostic counter: %s, labels=%s",
-                full_counter_name,
-                labels or {},
-            )
-        except Exception:
-            logger.debug("Failed to increment diagnostic counter: %s", counter_name, exc_info=True)
-
-    def handle(self, envelope: TelemetryEnvelope) -> None:
-        """Main entry point for processing telemetry envelopes.
-
-        Args:
-            envelope: The telemetry envelope to process.
-        """
-        # Check for duplicate events
-        if self._is_duplicate(envelope):
-            logger.debug(
-                "Skipping duplicate event: tenant_id=%s, event_id=%s",
-                envelope.tenant_id,
-                envelope.event_id,
-            )
-            self._increment_diagnostic_counter("deduped_total")
-            return
-
-        # Route to appropriate handler based on case
-        case = envelope.case
-        if case == TelemetryCase.APP_CREATED:
-            self._on_app_created(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "app_created"})
-        elif case == TelemetryCase.APP_UPDATED:
-            self._on_app_updated(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "app_updated"})
-        elif case == TelemetryCase.APP_DELETED:
-            self._on_app_deleted(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "app_deleted"})
-        elif case == TelemetryCase.FEEDBACK_CREATED:
-            self._on_feedback_created(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "feedback_created"})
-        elif case == TelemetryCase.MESSAGE_RUN:
-            self._on_message_run(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "message_run"})
-        elif case == TelemetryCase.TOOL_EXECUTION:
-            self._on_tool_execution(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "tool_execution"})
-        elif case == TelemetryCase.MODERATION_CHECK:
-            self._on_moderation_check(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "moderation_check"})
-        elif case == TelemetryCase.SUGGESTED_QUESTION:
-            self._on_suggested_question(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "suggested_question"})
-        elif case == TelemetryCase.DATASET_RETRIEVAL:
-            self._on_dataset_retrieval(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "dataset_retrieval"})
-        elif case == TelemetryCase.GENERATE_NAME:
-            self._on_generate_name(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "generate_name"})
-        elif case == TelemetryCase.PROMPT_GENERATION:
-            self._on_prompt_generation(envelope)
-            self._increment_diagnostic_counter("processed_total", {"case": "prompt_generation"})
-        else:
-            logger.warning(
-                "Unknown telemetry case: %s (tenant_id=%s, event_id=%s)",
-                case,
-                envelope.tenant_id,
-                envelope.event_id,
-            )
-
-    def _is_duplicate(self, envelope: TelemetryEnvelope) -> bool:
-        """Check if this event has already been processed.
-
-        Uses Redis with TTL for deduplication. Returns True if duplicate,
-        False if first time seeing this event.
-
-        Args:
-            envelope: The telemetry envelope to check.
-
-        Returns:
-            True if this event_id has been seen before, False otherwise.
-        """
-        dedup_key = f"telemetry:dedup:{envelope.tenant_id}:{envelope.event_id}"
-
-        try:
-            # Atomic set-if-not-exists with 1h TTL
-            # Returns True if key was set (first time), None if already exists (duplicate)
-            was_set = redis_client.set(dedup_key, b"1", nx=True, ex=3600)
-            return was_set is None
-        except Exception:
-            # Fail open: if Redis is unavailable, process the event
-            # (prefer occasional duplicate over lost data)
-            logger.warning(
-                "Redis unavailable for deduplication check, processing event anyway: %s",
-                envelope.event_id,
-                exc_info=True,
-            )
-            return False
-
-    def _rehydrate(self, envelope: TelemetryEnvelope) -> dict[str, Any]:
-        """Rehydrate payload from storage reference or inline data.
-
-        If the envelope payload is empty and metadata contains a
-        ``payload_ref``, the full payload is loaded from object storage
-        (where the gateway wrote it as JSON).  When both the inline
-        payload and storage resolution fail, a degraded-event marker
-        is emitted so the gap is observable.
-
-        Args:
-            envelope: The telemetry envelope containing payload data.
-
-        Returns:
-            The rehydrated payload dictionary, or ``{}`` on total failure.
-        """
-        payload = envelope.payload
-
-        # Resolve from object storage when the gateway offloaded a large payload.
-        if not payload and envelope.metadata:
-            payload_ref = envelope.metadata.get("payload_ref")
-            if payload_ref:
-                try:
-                    payload_bytes = storage.load(payload_ref)
-                    payload = json.loads(payload_bytes.decode("utf-8"))
-                    logger.debug("Loaded payload from storage: key=%s", payload_ref)
-                except Exception:
-                    logger.warning(
-                        "Failed to load payload from storage: key=%s, event_id=%s",
-                        payload_ref,
-                        envelope.event_id,
-                        exc_info=True,
-                    )
-
-        if not payload:
-            # Storage resolution failed or no data available — emit degraded event.
-            logger.error(
-                "Payload rehydration failed for event_id=%s, tenant_id=%s, case=%s",
-                envelope.event_id,
-                envelope.tenant_id,
-                envelope.case,
-            )
-            from enterprise.telemetry.entities import EnterpriseTelemetryEvent
-            from enterprise.telemetry.telemetry_log import emit_metric_only_event
-
-            emit_metric_only_event(
-                event_name=EnterpriseTelemetryEvent.REHYDRATION_FAILED,
-                attributes={
-                    "dify.tenant_id": envelope.tenant_id,
-                    "dify.event_id": envelope.event_id,
-                    "dify.case": envelope.case,
-                    "rehydration_failed": True,
-                },
-                tenant_id=envelope.tenant_id,
-            )
-            self._increment_diagnostic_counter("rehydration_failed_total")
-            return {}
-
-        return payload
-
-    # Stub methods for each metric/log case
-    # These will be implemented in later tasks with actual emission logic
-
-    def _on_app_created(self, envelope: TelemetryEnvelope) -> None:
-        """Handle app created event."""
-        from enterprise.telemetry.entities import EnterpriseTelemetryCounter, EnterpriseTelemetryEvent
-        from enterprise.telemetry.telemetry_log import emit_metric_only_event
-        from extensions.ext_enterprise_telemetry import get_enterprise_exporter
-
-        exporter = get_enterprise_exporter()
-        if not exporter:
-            logger.debug("No exporter available for APP_CREATED: event_id=%s", envelope.event_id)
-            return
-
-        payload = self._rehydrate(envelope)
-        if not payload:
-            return
-
-        attrs = {
-            "dify.app.id": payload.get("app_id"),
-            "dify.tenant_id": envelope.tenant_id,
-            "dify.event.id": envelope.event_id,
-            "dify.app.mode": payload.get("mode"),
-        }
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.APP_CREATED,
-            attributes=attrs,
-            tenant_id=envelope.tenant_id,
-        )
-        exporter.increment_counter(
-            EnterpriseTelemetryCounter.APP_CREATED,
-            1,
-            {
-                "tenant_id": envelope.tenant_id,
-                "app_id": str(payload.get("app_id", "")),
-                "mode": str(payload.get("mode", "")),
-            },
-        )
-
-    def _on_app_updated(self, envelope: TelemetryEnvelope) -> None:
-        """Handle app updated event."""
-        from enterprise.telemetry.entities import EnterpriseTelemetryCounter, EnterpriseTelemetryEvent
-        from enterprise.telemetry.telemetry_log import emit_metric_only_event
-        from extensions.ext_enterprise_telemetry import get_enterprise_exporter
-
-        exporter = get_enterprise_exporter()
-        if not exporter:
-            logger.debug("No exporter available for APP_UPDATED: event_id=%s", envelope.event_id)
-            return
-
-        payload = self._rehydrate(envelope)
-        if not payload:
-            return
-
-        attrs = {
-            "dify.app.id": payload.get("app_id"),
-            "dify.tenant_id": envelope.tenant_id,
-            "dify.event.id": envelope.event_id,
-        }
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.APP_UPDATED,
-            attributes=attrs,
-            tenant_id=envelope.tenant_id,
-        )
-        exporter.increment_counter(
-            EnterpriseTelemetryCounter.APP_UPDATED,
-            1,
-            {
-                "tenant_id": envelope.tenant_id,
-                "app_id": str(payload.get("app_id", "")),
-            },
-        )
-
-    def _on_app_deleted(self, envelope: TelemetryEnvelope) -> None:
-        """Handle app deleted event."""
-        from enterprise.telemetry.entities import EnterpriseTelemetryCounter, EnterpriseTelemetryEvent
-        from enterprise.telemetry.telemetry_log import emit_metric_only_event
-        from extensions.ext_enterprise_telemetry import get_enterprise_exporter
-
-        exporter = get_enterprise_exporter()
-        if not exporter:
-            logger.debug("No exporter available for APP_DELETED: event_id=%s", envelope.event_id)
-            return
-
-        payload = self._rehydrate(envelope)
-        if not payload:
-            return
-
-        attrs = {
-            "dify.app.id": payload.get("app_id"),
-            "dify.tenant_id": envelope.tenant_id,
-            "dify.event.id": envelope.event_id,
-        }
-
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.APP_DELETED,
-            attributes=attrs,
-            tenant_id=envelope.tenant_id,
-        )
-        exporter.increment_counter(
-            EnterpriseTelemetryCounter.APP_DELETED,
-            1,
-            {
-                "tenant_id": envelope.tenant_id,
-                "app_id": str(payload.get("app_id", "")),
-            },
-        )
-
-    def _on_feedback_created(self, envelope: TelemetryEnvelope) -> None:
-        """Handle feedback created event."""
-        from enterprise.telemetry.entities import EnterpriseTelemetryCounter, EnterpriseTelemetryEvent
-        from enterprise.telemetry.telemetry_log import emit_metric_only_event
-        from extensions.ext_enterprise_telemetry import get_enterprise_exporter
-
-        exporter = get_enterprise_exporter()
-        if not exporter:
-            logger.debug("No exporter available for FEEDBACK_CREATED: event_id=%s", envelope.event_id)
-            return
-
-        payload = self._rehydrate(envelope)
-        if not payload:
-            return
-
-        include_content = exporter.include_content
-        attrs: dict = {
-            "dify.message.id": payload.get("message_id"),
-            "dify.tenant_id": envelope.tenant_id,
-            "dify.event.id": envelope.event_id,
-            "dify.app_id": payload.get("app_id"),
-            "dify.conversation.id": payload.get("conversation_id"),
-            "gen_ai.user.id": payload.get("from_end_user_id") or payload.get("from_account_id"),
-            "dify.feedback.rating": payload.get("rating"),
-            "dify.feedback.from_source": payload.get("from_source"),
-        }
-        if include_content:
-            attrs["dify.feedback.content"] = payload.get("content")
-
-        user_id = payload.get("from_end_user_id") or payload.get("from_account_id")
-        emit_metric_only_event(
-            event_name=EnterpriseTelemetryEvent.FEEDBACK_CREATED,
-            attributes=attrs,
-            tenant_id=envelope.tenant_id,
-            user_id=str(user_id or ""),
-        )
-        exporter.increment_counter(
-            EnterpriseTelemetryCounter.FEEDBACK,
-            1,
-            {
-                "tenant_id": envelope.tenant_id,
-                "app_id": str(payload.get("app_id", "")),
-                "rating": str(payload.get("rating", "")),
-            },
-        )
-
-    def _on_message_run(self, envelope: TelemetryEnvelope) -> None:
-        """Handle message run event (stub)."""
-        logger.debug("Processing MESSAGE_RUN: event_id=%s", envelope.event_id)
-
-    def _on_tool_execution(self, envelope: TelemetryEnvelope) -> None:
-        """Handle tool execution event (stub)."""
-        logger.debug("Processing TOOL_EXECUTION: event_id=%s", envelope.event_id)
-
-    def _on_moderation_check(self, envelope: TelemetryEnvelope) -> None:
-        """Handle moderation check event (stub)."""
-        logger.debug("Processing MODERATION_CHECK: event_id=%s", envelope.event_id)
-
-    def _on_suggested_question(self, envelope: TelemetryEnvelope) -> None:
-        """Handle suggested question event (stub)."""
-        logger.debug("Processing SUGGESTED_QUESTION: event_id=%s", envelope.event_id)
-
-    def _on_dataset_retrieval(self, envelope: TelemetryEnvelope) -> None:
-        """Handle dataset retrieval event (stub)."""
-        logger.debug("Processing DATASET_RETRIEVAL: event_id=%s", envelope.event_id)
-
-    def _on_generate_name(self, envelope: TelemetryEnvelope) -> None:
-        """Handle generate name event (stub)."""
-        logger.debug("Processing GENERATE_NAME: event_id=%s", envelope.event_id)
-
-    def _on_prompt_generation(self, envelope: TelemetryEnvelope) -> None:
-        """Handle prompt generation event (stub)."""
-        logger.debug("Processing PROMPT_GENERATION: event_id=%s", envelope.event_id)

api/enterprise/telemetry/telemetry_log.py

@@ -1,122 +0,0 @@
-"""Structured-log emitter for enterprise telemetry events.
-
-Emits structured JSON log lines correlated with OTEL traces via trace_id.
-Picked up by ``StructuredJSONFormatter`` → stdout/Loki/Elastic.
-"""
-
-from __future__ import annotations
-
-import logging
-import uuid
-from functools import lru_cache
-from typing import TYPE_CHECKING, Any
-
-if TYPE_CHECKING:
-    from enterprise.telemetry.entities import EnterpriseTelemetryEvent
-
-logger = logging.getLogger("dify.telemetry")
-
-
-@lru_cache(maxsize=4096)
-def compute_trace_id_hex(uuid_str: str | None) -> str:
-    """Convert a business UUID string to a 32-hex OTEL-compatible trace_id.
-
-    Returns empty string when *uuid_str* is ``None`` or invalid.
-    """
-    if not uuid_str:
-        return ""
-    normalized = uuid_str.strip().lower()
-    if len(normalized) == 32 and all(ch in "0123456789abcdef" for ch in normalized):
-        return normalized
-    try:
-        return f"{uuid.UUID(normalized).int:032x}"
-    except (ValueError, AttributeError):
-        return ""
-
-
-@lru_cache(maxsize=4096)
-def compute_span_id_hex(uuid_str: str | None) -> str:
-    if not uuid_str:
-        return ""
-    normalized = uuid_str.strip().lower()
-    if len(normalized) == 16 and all(ch in "0123456789abcdef" for ch in normalized):
-        return normalized
-    try:
-        from enterprise.telemetry.id_generator import compute_deterministic_span_id
-
-        return f"{compute_deterministic_span_id(normalized):016x}"
-    except (ValueError, AttributeError):
-        return ""
-
-
-def emit_telemetry_log(
-    *,
-    event_name: str | EnterpriseTelemetryEvent,
-    attributes: dict[str, Any],
-    signal: str = "metric_only",
-    trace_id_source: str | None = None,
-    span_id_source: str | None = None,
-    tenant_id: str | None = None,
-    user_id: str | None = None,
-) -> None:
-    """Emit a structured log line for a telemetry event.
-
-    Parameters
-    ----------
-    event_name:
-        Canonical event name, e.g. ``"dify.workflow.run"``.
-    attributes:
-        All event-specific attributes (already built by the caller).
-    signal:
-        ``"metric_only"`` for events with no span, ``"span_detail"``
-        for detail logs accompanying a slim span.
-    trace_id_source:
-        A UUID string (e.g. ``workflow_run_id``) used to derive a 32-hex
-        trace_id for cross-signal correlation.
-    tenant_id:
-        Tenant identifier (for the ``IdentityContextFilter``).
-    user_id:
-        User identifier (for the ``IdentityContextFilter``).
-    """
-    if not logger.isEnabledFor(logging.INFO):
-        return
-    attrs = {
-        "dify.event.name": event_name,
-        "dify.event.signal": signal,
-        **attributes,
-    }
-
-    extra: dict[str, Any] = {"attributes": attrs}
-
-    trace_id_hex = compute_trace_id_hex(trace_id_source)
-    if trace_id_hex:
-        extra["trace_id"] = trace_id_hex
-    span_id_hex = compute_span_id_hex(span_id_source)
-    if span_id_hex:
-        extra["span_id"] = span_id_hex
-    if tenant_id:
-        extra["tenant_id"] = tenant_id
-    if user_id:
-        extra["user_id"] = user_id
-
-    logger.info("telemetry.%s", signal, extra=extra)
-
-
-def emit_metric_only_event(
-    *,
-    event_name: str | EnterpriseTelemetryEvent,
-    attributes: dict[str, Any],
-    trace_id_source: str | None = None,
-    span_id_source: str | None = None,
-    tenant_id: str | None = None,
-    user_id: str | None = None,
-) -> None:
-    emit_telemetry_log(
-        event_name=event_name,
-        attributes=attributes,
-        signal="metric_only",
-        trace_id_source=trace_id_source,
-        span_id_source=span_id_source,
-        tenant_id=tenant_id,
-        user_id=user_id,
-    )

api/events/app_event.py

@@ -3,12 +3,6 @@ from blinker import signal
 # sender: app
 app_was_created = signal("app-was-created")
 
-# sender: app
-app_was_deleted = signal("app-was-deleted")
-
-# sender: app
-app_was_updated = signal("app-was-updated")
-
 # sender: app, kwargs: app_model_config
 app_model_config_was_updated = signal("app-model-config-was-updated")
 

api/events/feedback_event.py

@@ -1,4 +0,0 @@
-from blinker import signal
-
-# sender: MessageFeedback, kwargs: tenant_id
-feedback_was_created = signal("feedback-was-created")

api/extensions/ext_celery.py

@@ -184,8 +184,6 @@ def init_app(app: DifyApp) -> Celery:
             "task": "schedule.trigger_provider_refresh_task.trigger_provider_refresh",
             "schedule": timedelta(minutes=dify_config.TRIGGER_PROVIDER_REFRESH_INTERVAL),
         }
-    if dify_config.ENTERPRISE_TELEMETRY_ENABLED:
-        imports.append("tasks.enterprise_telemetry_task")
     celery_app.conf.update(beat_schedule=beat_schedule, imports=imports)
 
     return celery_app

api/extensions/ext_enterprise_telemetry.py

@@ -1,50 +0,0 @@
-"""Flask extension for enterprise telemetry lifecycle management.
-
-Initializes the EnterpriseExporter singleton during ``create_app()``
-(single-threaded), registers blinker event handlers, and hooks atexit
-for graceful shutdown.
-
-Skipped entirely when ``ENTERPRISE_ENABLED`` and ``ENTERPRISE_TELEMETRY_ENABLED``
-are false (``is_enabled()`` gate).
-"""
-
-from __future__ import annotations
-
-import atexit
-import logging
-from typing import TYPE_CHECKING
-
-from configs import dify_config
-
-if TYPE_CHECKING:
-    from dify_app import DifyApp
-    from enterprise.telemetry.exporter import EnterpriseExporter
-
-logger = logging.getLogger(__name__)
-
-_exporter: EnterpriseExporter | None = None
-
-
-def is_enabled() -> bool:
-    return bool(dify_config.ENTERPRISE_ENABLED and dify_config.ENTERPRISE_TELEMETRY_ENABLED)
-
-
-def init_app(app: DifyApp) -> None:
-    global _exporter
-
-    if not is_enabled():
-        return
-
-    from enterprise.telemetry.exporter import EnterpriseExporter
-
-    _exporter = EnterpriseExporter(dify_config)
-    atexit.register(_exporter.shutdown)
-
-    # Import to trigger @signal.connect decorator registration
-    import enterprise.telemetry.event_handlers  # noqa: F401  # type: ignore[reportUnusedImport]
-
-    logger.info("Enterprise telemetry initialized")
-
-
-def get_enterprise_exporter() -> EnterpriseExporter | None:
-    return _exporter

api/extensions/ext_otel.py

@@ -59,24 +59,16 @@ def init_app(app: DifyApp):
     protocol = (dify_config.OTEL_EXPORTER_OTLP_PROTOCOL or "").lower()
     if dify_config.OTEL_EXPORTER_TYPE == "otlp":
         if protocol == "grpc":
-            # Auto-detect TLS: https:// uses secure, everything else is insecure
-            endpoint = dify_config.OTLP_BASE_ENDPOINT
-            insecure = not endpoint.startswith("https://")
-
             exporter = GRPCSpanExporter(
-                endpoint=endpoint,
+                endpoint=dify_config.OTLP_BASE_ENDPOINT,
                 # Header field names must consist of lowercase letters, check RFC7540
-                headers=(
-                    (("authorization", f"Bearer {dify_config.OTLP_API_KEY}"),) if dify_config.OTLP_API_KEY else None
-                ),
-                insecure=insecure,
+                headers=(("authorization", f"Bearer {dify_config.OTLP_API_KEY}"),),
+                insecure=True,
             )
             metric_exporter = GRPCMetricExporter(
-                endpoint=endpoint,
-                headers=(
-                    (("authorization", f"Bearer {dify_config.OTLP_API_KEY}"),) if dify_config.OTLP_API_KEY else None
-                ),
-                insecure=insecure,
+                endpoint=dify_config.OTLP_BASE_ENDPOINT,
+                headers=(("authorization", f"Bearer {dify_config.OTLP_API_KEY}"),),
+                insecure=True,
             )
         else:
             headers = {"Authorization": f"Bearer {dify_config.OTLP_API_KEY}"} if dify_config.OTLP_API_KEY else None

api/extensions/otel/parser/__init__.py

@@ -5,7 +5,7 @@ This module provides parsers that extract node-specific metadata and set
 OpenTelemetry span attributes according to semantic conventions.
 """
 
-from extensions.otel.parser.base import DefaultNodeOTelParser, NodeOTelParser, safe_json_dumps, should_include_content
+from extensions.otel.parser.base import DefaultNodeOTelParser, NodeOTelParser, safe_json_dumps
 from extensions.otel.parser.llm import LLMNodeOTelParser
 from extensions.otel.parser.retrieval import RetrievalNodeOTelParser
 from extensions.otel.parser.tool import ToolNodeOTelParser
@@ -17,5 +17,4 @@ __all__ = [
     "RetrievalNodeOTelParser",
     "ToolNodeOTelParser",
     "safe_json_dumps",
-    "should_include_content",
 ]

api/extensions/otel/parser/base.py

@@ -1,10 +1,5 @@
 """
 Base parser interface and utilities for OpenTelemetry node parsers.
-
-Content gating: ``should_include_content()`` controls whether content-bearing
-span attributes (inputs, outputs, prompts, completions, documents) are written.
-Gate is only active in EE (``ENTERPRISE_ENABLED=True``) when
-``ENTERPRISE_INCLUDE_CONTENT=False``; CE behaviour is unchanged.
 """
 
 import json
@@ -14,7 +9,6 @@ from opentelemetry.trace import Span
 from opentelemetry.trace.status import Status, StatusCode
 from pydantic import BaseModel
 
-from configs import dify_config
 from core.file.models import File
 from core.variables import Segment
 from core.workflow.enums import NodeType
@@ -23,17 +17,6 @@ from core.workflow.nodes.base.node import Node
 from extensions.otel.semconv.gen_ai import ChainAttributes, GenAIAttributes
 
 
-def should_include_content() -> bool:
-    """Return True if content should be written to spans.
-
-    CE (ENTERPRISE_ENABLED=False): always True — no behaviour change.
-    EE: follows ENTERPRISE_INCLUDE_CONTENT (default True).
-    """
-    if not dify_config.ENTERPRISE_ENABLED:
-        return True
-    return dify_config.ENTERPRISE_INCLUDE_CONTENT
-
-
 def safe_json_dumps(obj: Any, ensure_ascii: bool = False) -> str:
     """
     Safely serialize objects to JSON, handling non-serializable types.
@@ -122,11 +105,10 @@ class DefaultNodeOTelParser:
         # Extract inputs and outputs from result_event
         if result_event and result_event.node_run_result:
             node_run_result = result_event.node_run_result
-            if should_include_content():
-                if node_run_result.inputs:
-                    span.set_attribute(ChainAttributes.INPUT_VALUE, safe_json_dumps(node_run_result.inputs))
-                if node_run_result.outputs:
-                    span.set_attribute(ChainAttributes.OUTPUT_VALUE, safe_json_dumps(node_run_result.outputs))
+            if node_run_result.inputs:
+                span.set_attribute(ChainAttributes.INPUT_VALUE, safe_json_dumps(node_run_result.inputs))
+            if node_run_result.outputs:
+                span.set_attribute(ChainAttributes.OUTPUT_VALUE, safe_json_dumps(node_run_result.outputs))
 
         if error:
             span.record_exception(error)

api/extensions/otel/parser/llm.py

@@ -10,7 +10,7 @@ from opentelemetry.trace import Span
 
 from core.workflow.graph_events import GraphNodeEventBase
 from core.workflow.nodes.base.node import Node
-from extensions.otel.parser.base import DefaultNodeOTelParser, safe_json_dumps, should_include_content
+from extensions.otel.parser.base import DefaultNodeOTelParser, safe_json_dumps
 from extensions.otel.semconv.gen_ai import LLMAttributes
 
 logger = logging.getLogger(__name__)
@@ -132,19 +132,24 @@ class LLMNodeOTelParser:
             span.set_attribute(LLMAttributes.USAGE_OUTPUT_TOKENS, completion_tokens)
             span.set_attribute(LLMAttributes.USAGE_TOTAL_TOKENS, total_tokens)
 
-        # Prompts and completion — gated by content policy
-        if should_include_content():
-            prompts = process_data.get("prompts", [])
-            if prompts:
-                prompts_json = safe_json_dumps(prompts)
-                span.set_attribute(LLMAttributes.PROMPT, prompts_json)
+        # Prompts and completion
+        prompts = process_data.get("prompts", [])
+        if prompts:
+            prompts_json = safe_json_dumps(prompts)
+            span.set_attribute(LLMAttributes.PROMPT, prompts_json)
 
-            text_output = str(outputs.get("text", ""))
-            if text_output:
-                span.set_attribute(LLMAttributes.COMPLETION, text_output)
+        text_output = str(outputs.get("text", ""))
+        if text_output:
+            span.set_attribute(LLMAttributes.COMPLETION, text_output)
 
-            # Structured input/output messages
-            gen_ai_input_message = _format_input_messages(process_data)
-            gen_ai_output_message = _format_output_messages(outputs)
-            span.set_attribute(LLMAttributes.INPUT_MESSAGE, gen_ai_input_message)
-            span.set_attribute(LLMAttributes.OUTPUT_MESSAGE, gen_ai_output_message)
+        # Finish reason
+        finish_reason = outputs.get("finish_reason") or ""
+        if finish_reason:
+            span.set_attribute(LLMAttributes.RESPONSE_FINISH_REASON, finish_reason)
+
+        # Structured input/output messages
+        gen_ai_input_message = _format_input_messages(process_data)
+        gen_ai_output_message = _format_output_messages(outputs)
+
+        span.set_attribute(LLMAttributes.INPUT_MESSAGE, gen_ai_input_message)
+        span.set_attribute(LLMAttributes.OUTPUT_MESSAGE, gen_ai_output_message)

api/extensions/otel/parser/retrieval.py

@@ -11,7 +11,7 @@ from opentelemetry.trace import Span
 from core.variables import Segment
 from core.workflow.graph_events import GraphNodeEventBase
 from core.workflow.nodes.base.node import Node
-from extensions.otel.parser.base import DefaultNodeOTelParser, safe_json_dumps, should_include_content
+from extensions.otel.parser.base import DefaultNodeOTelParser, safe_json_dumps
 from extensions.otel.semconv.gen_ai import RetrieverAttributes
 
 logger = logging.getLogger(__name__)
@@ -83,21 +83,23 @@ class RetrievalNodeOTelParser:
         inputs = node_run_result.inputs or {}
         outputs = node_run_result.outputs or {}
 
-        # Query and documents — gated by content policy
-        if should_include_content():
-            query = str(inputs.get("query", "")) if inputs else ""
-            if query:
-                span.set_attribute(RetrieverAttributes.QUERY, query)
+        # Extract query from inputs
+        query = str(inputs.get("query", "")) if inputs else ""
+        if query:
+            span.set_attribute(RetrieverAttributes.QUERY, query)
 
-            result_value = outputs.get("result") if outputs else None
-            retrieval_documents: list[Any] = []
-            if result_value:
-                value_to_check = result_value
-                if isinstance(result_value, Segment):
-                    value_to_check = result_value.value
-                if isinstance(value_to_check, (list, Sequence)):
-                    retrieval_documents = list(value_to_check)
-            if retrieval_documents:
-                semantic_retrieval_documents = _format_retrieval_documents(retrieval_documents)
-                semantic_retrieval_documents_json = safe_json_dumps(semantic_retrieval_documents)
-                span.set_attribute(RetrieverAttributes.DOCUMENT, semantic_retrieval_documents_json)
+        # Extract and format retrieval documents from outputs
+        result_value = outputs.get("result") if outputs else None
+        retrieval_documents: list[Any] = []
+        if result_value:
+            value_to_check = result_value
+            if isinstance(result_value, Segment):
+                value_to_check = result_value.value
+
+            if isinstance(value_to_check, (list, Sequence)):
+                retrieval_documents = list(value_to_check)
+
+        if retrieval_documents:
+            semantic_retrieval_documents = _format_retrieval_documents(retrieval_documents)
+            semantic_retrieval_documents_json = safe_json_dumps(semantic_retrieval_documents)
+            span.set_attribute(RetrieverAttributes.DOCUMENT, semantic_retrieval_documents_json)

api/extensions/otel/parser/tool.py

@@ -8,7 +8,7 @@ from core.workflow.enums import WorkflowNodeExecutionMetadataKey
 from core.workflow.graph_events import GraphNodeEventBase
 from core.workflow.nodes.base.node import Node
 from core.workflow.nodes.tool.entities import ToolNodeData
-from extensions.otel.parser.base import DefaultNodeOTelParser, safe_json_dumps, should_include_content
+from extensions.otel.parser.base import DefaultNodeOTelParser, safe_json_dumps
 from extensions.otel.semconv.gen_ai import ToolAttributes
 
 
@@ -40,14 +40,8 @@ class ToolNodeOTelParser:
         if tool_info:
             span.set_attribute(ToolAttributes.TOOL_DESCRIPTION, safe_json_dumps(tool_info))
 
-        # Tool call arguments and result — gated by content policy
-        if should_include_content():
-            if result_event and result_event.node_run_result and result_event.node_run_result.inputs:
-                span.set_attribute(
-                    ToolAttributes.TOOL_CALL_ARGUMENTS, safe_json_dumps(result_event.node_run_result.inputs)
-                )
+        if result_event and result_event.node_run_result and result_event.node_run_result.inputs:
+            span.set_attribute(ToolAttributes.TOOL_CALL_ARGUMENTS, safe_json_dumps(result_event.node_run_result.inputs))
 
-            if result_event and result_event.node_run_result and result_event.node_run_result.outputs:
-                span.set_attribute(
-                    ToolAttributes.TOOL_CALL_RESULT, safe_json_dumps(result_event.node_run_result.outputs)
-                )
+        if result_event and result_event.node_run_result and result_event.node_run_result.outputs:
+            span.set_attribute(ToolAttributes.TOOL_CALL_RESULT, safe_json_dumps(result_event.node_run_result.outputs))

api/extensions/otel/semconv/dify.py

@@ -21,15 +21,3 @@ class DifySpanAttributes:
 
     INVOKE_FROM = "dify.invoke_from"
     """Invocation source, e.g. SERVICE_API, WEB_APP, DEBUGGER."""
-
-    INVOKED_BY = "dify.invoked_by"
-    """Invoked by, e.g. end_user, account, user."""
-
-    USAGE_INPUT_TOKENS = "gen_ai.usage.input_tokens"
-    """Number of input tokens (prompt tokens) used."""
-
-    USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens"
-    """Number of output tokens (completion tokens) generated."""
-
-    USAGE_TOTAL_TOKENS = "gen_ai.usage.total_tokens"
-    """Total number of tokens used."""

api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py

@@ -390,7 +390,8 @@ class ClickZettaVolumeStorage(BaseStorage):
         """
         content = self.load_once(filename)
 
-        Path(target_filepath).write_bytes(content)
+        with Path(target_filepath).open("wb") as f:
+            f.write(content)
 
         logger.debug("File %s downloaded from ClickZetta Volume to %s", filename, target_filepath)
 

api/fields/annotation_fields.py

@@ -1,69 +1,36 @@
-from __future__ import annotations
+from flask_restx import Namespace, fields
 
-from datetime import datetime
+from libs.helper import TimestampField
 
-from pydantic import BaseModel, ConfigDict, Field, field_validator
+annotation_fields = {
+    "id": fields.String,
+    "question": fields.String,
+    "answer": fields.Raw(attribute="content"),
+    "hit_count": fields.Integer,
+    "created_at": TimestampField,
+    # 'account': fields.Nested(simple_account_fields, allow_null=True)
+}
 
 
-def _to_timestamp(value: datetime | int | None) -> int | None:
-    if isinstance(value, datetime):
-        return int(value.timestamp())
-    return value
+def build_annotation_model(api_or_ns: Namespace):
+    """Build the annotation model for the API or Namespace."""
+    return api_or_ns.model("Annotation", annotation_fields)
 
 
-class ResponseModel(BaseModel):
-    model_config = ConfigDict(
-        from_attributes=True,
-        extra="ignore",
-        populate_by_name=True,
-        serialize_by_alias=True,
-        protected_namespaces=(),
-    )
+annotation_list_fields = {
+    "data": fields.List(fields.Nested(annotation_fields)),
+}
 
+annotation_hit_history_fields = {
+    "id": fields.String,
+    "source": fields.String,
+    "score": fields.Float,
+    "question": fields.String,
+    "created_at": TimestampField,
+    "match": fields.String(attribute="annotation_question"),
+    "response": fields.String(attribute="annotation_content"),
+}
 
-class Annotation(ResponseModel):
-    id: str
-    question: str | None = None
-    answer: str | None = Field(default=None, validation_alias="content")
-    hit_count: int | None = None
-    created_at: int | None = None
-
-    @field_validator("created_at", mode="before")
-    @classmethod
-    def _normalize_created_at(cls, value: datetime | int | None) -> int | None:
-        return _to_timestamp(value)
-
-
-class AnnotationList(ResponseModel):
-    data: list[Annotation]
-    has_more: bool
-    limit: int
-    total: int
-    page: int
-
-
-class AnnotationExportList(ResponseModel):
-    data: list[Annotation]
-
-
-class AnnotationHitHistory(ResponseModel):
-    id: str
-    source: str | None = None
-    score: float | None = None
-    question: str | None = None
-    created_at: int | None = None
-    match: str | None = Field(default=None, validation_alias="annotation_question")
-    response: str | None = Field(default=None, validation_alias="annotation_content")
-
-    @field_validator("created_at", mode="before")
-    @classmethod
-    def _normalize_created_at(cls, value: datetime | int | None) -> int | None:
-        return _to_timestamp(value)
-
-
-class AnnotationHitHistoryList(ResponseModel):
-    data: list[AnnotationHitHistory]
-    has_more: bool
-    limit: int
-    total: int
-    page: int
+annotation_hit_history_list_fields = {
+    "data": fields.List(fields.Nested(annotation_hit_history_fields)),
+}

api/fields/end_user_fields.py

@@ -1,7 +1,4 @@
-from __future__ import annotations
-
-from flask_restx import fields
-from pydantic import BaseModel, ConfigDict
+from flask_restx import Namespace, fields
 
 simple_end_user_fields = {
     "id": fields.String,
@@ -11,18 +8,5 @@ simple_end_user_fields = {
 }
 
 
-class ResponseModel(BaseModel):
-    model_config = ConfigDict(
-        from_attributes=True,
-        extra="ignore",
-        populate_by_name=True,
-        serialize_by_alias=True,
-        protected_namespaces=(),
-    )
-
-
-class SimpleEndUser(ResponseModel):
-    id: str
-    type: str
-    is_anonymous: bool
-    session_id: str | None = None
+def build_simple_end_user_model(api_or_ns: Namespace):
+    return api_or_ns.model("SimpleEndUser", simple_end_user_fields)

api/fields/member_fields.py

@@ -1,11 +1,6 @@
-from __future__ import annotations
+from flask_restx import Namespace, fields
 
-from datetime import datetime
-
-from flask_restx import fields
-from pydantic import BaseModel, ConfigDict, computed_field, field_validator
-
-from core.file import helpers as file_helpers
+from libs.helper import AvatarUrlField, TimestampField
 
 simple_account_fields = {
     "id": fields.String,
@@ -14,78 +9,36 @@ simple_account_fields = {
 }
 
 
-def _to_timestamp(value: datetime | int | None) -> int | None:
-    if isinstance(value, datetime):
-        return int(value.timestamp())
-    return value
+def build_simple_account_model(api_or_ns: Namespace):
+    return api_or_ns.model("SimpleAccount", simple_account_fields)
 
 
-def _build_avatar_url(avatar: str | None) -> str | None:
-    if avatar is None:
-        return None
-    if avatar.startswith(("http://", "https://")):
-        return avatar
-    return file_helpers.get_signed_file_url(avatar)
+account_fields = {
+    "id": fields.String,
+    "name": fields.String,
+    "avatar": fields.String,
+    "avatar_url": AvatarUrlField,
+    "email": fields.String,
+    "is_password_set": fields.Boolean,
+    "interface_language": fields.String,
+    "interface_theme": fields.String,
+    "timezone": fields.String,
+    "last_login_at": TimestampField,
+    "last_login_ip": fields.String,
+    "created_at": TimestampField,
+}
 
+account_with_role_fields = {
+    "id": fields.String,
+    "name": fields.String,
+    "avatar": fields.String,
+    "avatar_url": AvatarUrlField,
+    "email": fields.String,
+    "last_login_at": TimestampField,
+    "last_active_at": TimestampField,
+    "created_at": TimestampField,
+    "role": fields.String,
+    "status": fields.String,
+}
 
-class ResponseModel(BaseModel):
-    model_config = ConfigDict(
-        from_attributes=True,
-        extra="ignore",
-        populate_by_name=True,
-        serialize_by_alias=True,
-        protected_namespaces=(),
-    )
-
-
-class SimpleAccount(ResponseModel):
-    id: str
-    name: str
-    email: str
-
-
-class _AccountAvatar(ResponseModel):
-    avatar: str | None = None
-
-    @computed_field(return_type=str | None)  # type: ignore[prop-decorator]
-    @property
-    def avatar_url(self) -> str | None:
-        return _build_avatar_url(self.avatar)
-
-
-class Account(_AccountAvatar):
-    id: str
-    name: str
-    email: str
-    is_password_set: bool
-    interface_language: str | None = None
-    interface_theme: str | None = None
-    timezone: str | None = None
-    last_login_at: int | None = None
-    last_login_ip: str | None = None
-    created_at: int | None = None
-
-    @field_validator("last_login_at", "created_at", mode="before")
-    @classmethod
-    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
-        return _to_timestamp(value)
-
-
-class AccountWithRole(_AccountAvatar):
-    id: str
-    name: str
-    email: str
-    last_login_at: int | None = None
-    last_active_at: int | None = None
-    created_at: int | None = None
-    role: str
-    status: str
-
-    @field_validator("last_login_at", "last_active_at", "created_at", mode="before")
-    @classmethod
-    def _normalize_timestamp(cls, value: datetime | int | None) -> int | None:
-        return _to_timestamp(value)
-
-
-class AccountWithRoleList(ResponseModel):
-    accounts: list[AccountWithRole]
+account_with_role_list_fields = {"accounts": fields.List(fields.Nested(account_with_role_fields))}

api/fields/tag_fields.py

@@ -1,20 +1,12 @@
-from __future__ import annotations
+from flask_restx import Namespace, fields
 
-from pydantic import BaseModel, ConfigDict
+dataset_tag_fields = {
+    "id": fields.String,
+    "name": fields.String,
+    "type": fields.String,
+    "binding_count": fields.String,
+}
 
 
-class ResponseModel(BaseModel):
-    model_config = ConfigDict(
-        from_attributes=True,
-        extra="ignore",
-        populate_by_name=True,
-        serialize_by_alias=True,
-        protected_namespaces=(),
-    )
-
-
-class DataSetTag(ResponseModel):
-    id: str
-    name: str
-    type: str
-    binding_count: str | None = None
+def build_dataset_tag_fields(api_or_ns: Namespace):
+    return api_or_ns.model("DataSetTag", dataset_tag_fields)

api/fields/workflow_app_log_fields.py

@@ -1,7 +1,7 @@
 from flask_restx import Namespace, fields
 
-from fields.end_user_fields import simple_end_user_fields
-from fields.member_fields import simple_account_fields
+from fields.end_user_fields import build_simple_end_user_model, simple_end_user_fields
+from fields.member_fields import build_simple_account_model, simple_account_fields
 from fields.workflow_run_fields import (
     build_workflow_run_for_archived_log_model,
     build_workflow_run_for_log_model,
@@ -25,9 +25,17 @@ workflow_app_log_partial_fields = {
 def build_workflow_app_log_partial_model(api_or_ns: Namespace):
     """Build the workflow app log partial model for the API or Namespace."""
     workflow_run_model = build_workflow_run_for_log_model(api_or_ns)
+    simple_account_model = build_simple_account_model(api_or_ns)
+    simple_end_user_model = build_simple_end_user_model(api_or_ns)
 
     copied_fields = workflow_app_log_partial_fields.copy()
     copied_fields["workflow_run"] = fields.Nested(workflow_run_model, attribute="workflow_run", allow_null=True)
+    copied_fields["created_by_account"] = fields.Nested(
+        simple_account_model, attribute="created_by_account", allow_null=True
+    )
+    copied_fields["created_by_end_user"] = fields.Nested(
+        simple_end_user_model, attribute="created_by_end_user", allow_null=True
+    )
     return api_or_ns.model("WorkflowAppLogPartial", copied_fields)
 
 
@@ -44,9 +52,17 @@ workflow_archived_log_partial_fields = {
 def build_workflow_archived_log_partial_model(api_or_ns: Namespace):
     """Build the workflow archived log partial model for the API or Namespace."""
     workflow_run_model = build_workflow_run_for_archived_log_model(api_or_ns)
+    simple_account_model = build_simple_account_model(api_or_ns)
+    simple_end_user_model = build_simple_end_user_model(api_or_ns)
 
     copied_fields = workflow_archived_log_partial_fields.copy()
     copied_fields["workflow_run"] = fields.Nested(workflow_run_model, allow_null=True)
+    copied_fields["created_by_account"] = fields.Nested(
+        simple_account_model, attribute="created_by_account", allow_null=True
+    )
+    copied_fields["created_by_end_user"] = fields.Nested(
+        simple_end_user_model, attribute="created_by_end_user", allow_null=True
+    )
     return api_or_ns.model("WorkflowArchivedLogPartial", copied_fields)
 
 

api/libs/db_migration_lock.py

@@ -1,213 +0,0 @@
-"""
-DB migration Redis lock with heartbeat renewal.
-
-This is intentionally migration-specific. Background renewal is a trade-off that makes sense
-for unbounded, blocking operations like DB migrations (DDL/DML) where the main thread cannot
-periodically refresh the lock TTL.
-
-Do NOT use this as a general-purpose lock primitive for normal application code. Prefer explicit
-lock lifecycle management (e.g. redis-py Lock context manager + `extend()` / `reacquire()` from
-the same thread) when execution flow is under control.
-"""
-
-from __future__ import annotations
-
-import logging
-import threading
-from typing import Any
-
-from redis.exceptions import LockNotOwnedError, RedisError
-
-logger = logging.getLogger(__name__)
-
-MIN_RENEW_INTERVAL_SECONDS = 0.1
-DEFAULT_RENEW_INTERVAL_DIVISOR = 3
-MIN_JOIN_TIMEOUT_SECONDS = 0.5
-MAX_JOIN_TIMEOUT_SECONDS = 5.0
-JOIN_TIMEOUT_MULTIPLIER = 2.0
-
-
-class DbMigrationAutoRenewLock:
-    """
-    Redis lock wrapper that automatically renews TTL while held (migration-only).
-
-    Notes:
-    - We force `thread_local=False` when creating the underlying redis-py lock, because the
-      lock token must be accessible from the heartbeat thread for `reacquire()` to work.
-    - `release_safely()` is best-effort: it never raises, so it won't mask the caller's
-      primary error/exit code.
-    """
-
-    _redis_client: Any
-    _name: str
-    _ttl_seconds: float
-    _renew_interval_seconds: float
-    _log_context: str | None
-    _logger: logging.Logger
-
-    _lock: Any
-    _stop_event: threading.Event | None
-    _thread: threading.Thread | None
-    _acquired: bool
-
-    def __init__(
-        self,
-        redis_client: Any,
-        name: str,
-        ttl_seconds: float = 60,
-        renew_interval_seconds: float | None = None,
-        *,
-        logger: logging.Logger | None = None,
-        log_context: str | None = None,
-    ) -> None:
-        self._redis_client = redis_client
-        self._name = name
-        self._ttl_seconds = float(ttl_seconds)
-        self._renew_interval_seconds = (
-            float(renew_interval_seconds)
-            if renew_interval_seconds is not None
-            else max(MIN_RENEW_INTERVAL_SECONDS, self._ttl_seconds / DEFAULT_RENEW_INTERVAL_DIVISOR)
-        )
-        self._logger = logger or logging.getLogger(__name__)
-        self._log_context = log_context
-
-        self._lock = None
-        self._stop_event = None
-        self._thread = None
-        self._acquired = False
-
-    @property
-    def name(self) -> str:
-        return self._name
-
-    def acquire(self, *args: Any, **kwargs: Any) -> bool:
-        """
-        Acquire the lock and start heartbeat renewal on success.
-
-        Accepts the same args/kwargs as redis-py `Lock.acquire()`.
-        """
-        # Prevent accidental double-acquire which could leave the previous heartbeat thread running.
-        if self._acquired:
-            raise RuntimeError("DB migration lock is already acquired; call release_safely() before acquiring again.")
-
-        # Reuse the lock object if we already created one.
-        if self._lock is None:
-            self._lock = self._redis_client.lock(
-                name=self._name,
-                timeout=self._ttl_seconds,
-                thread_local=False,
-            )
-        acquired = bool(self._lock.acquire(*args, **kwargs))
-        self._acquired = acquired
-        if acquired:
-            self._start_heartbeat()
-        return acquired
-
-    def owned(self) -> bool:
-        if self._lock is None:
-            return False
-        try:
-            return bool(self._lock.owned())
-        except Exception:
-            # Ownership checks are best-effort and must not break callers.
-            return False
-
-    def _start_heartbeat(self) -> None:
-        if self._lock is None:
-            return
-        if self._stop_event is not None:
-            return
-
-        self._stop_event = threading.Event()
-        self._thread = threading.Thread(
-            target=self._heartbeat_loop,
-            args=(self._lock, self._stop_event),
-            daemon=True,
-            name=f"DbMigrationAutoRenewLock({self._name})",
-        )
-        self._thread.start()
-
-    def _heartbeat_loop(self, lock: Any, stop_event: threading.Event) -> None:
-        while not stop_event.wait(self._renew_interval_seconds):
-            try:
-                lock.reacquire()
-            except LockNotOwnedError:
-                self._logger.warning(
-                    "DB migration lock is no longer owned during heartbeat; stop renewing. log_context=%s",
-                    self._log_context,
-                    exc_info=True,
-                )
-                return
-            except RedisError:
-                self._logger.warning(
-                    "Failed to renew DB migration lock due to Redis error; will retry. log_context=%s",
-                    self._log_context,
-                    exc_info=True,
-                )
-            except Exception:
-                self._logger.warning(
-                    "Unexpected error while renewing DB migration lock; will retry. log_context=%s",
-                    self._log_context,
-                    exc_info=True,
-                )
-
-    def release_safely(self, *, status: str | None = None) -> None:
-        """
-        Stop heartbeat and release lock. Never raises.
-
-        Args:
-            status: Optional caller-provided status (e.g. 'successful'/'failed') to add context to logs.
-        """
-        lock = self._lock
-        if lock is None:
-            return
-
-        self._stop_heartbeat()
-
-        # Lock release errors should never mask the real error/exit code.
-        try:
-            lock.release()
-        except LockNotOwnedError:
-            self._logger.warning(
-                "DB migration lock not owned on release; ignoring. status=%s log_context=%s",
-                status,
-                self._log_context,
-                exc_info=True,
-            )
-        except RedisError:
-            self._logger.warning(
-                "Failed to release DB migration lock due to Redis error; ignoring. status=%s log_context=%s",
-                status,
-                self._log_context,
-                exc_info=True,
-            )
-        except Exception:
-            self._logger.warning(
-                "Unexpected error while releasing DB migration lock; ignoring. status=%s log_context=%s",
-                status,
-                self._log_context,
-                exc_info=True,
-            )
-        finally:
-            self._acquired = False
-            self._lock = None
-
-    def _stop_heartbeat(self) -> None:
-        if self._stop_event is None:
-            return
-        self._stop_event.set()
-        if self._thread is not None:
-            # Best-effort join: if Redis calls are blocked, the daemon thread may remain alive.
-            join_timeout_seconds = max(
-                MIN_JOIN_TIMEOUT_SECONDS,
-                min(MAX_JOIN_TIMEOUT_SECONDS, self._renew_interval_seconds * JOIN_TIMEOUT_MULTIPLIER),
-            )
-            self._thread.join(timeout=join_timeout_seconds)
-            if self._thread.is_alive():
-                self._logger.warning(
-                    "DB migration lock heartbeat thread did not stop within %.2fs; ignoring. log_context=%s",
-                    join_timeout_seconds,
-                    self._log_context,
-                )
-        self._stop_event = None
-        self._thread = None

api/migrations/versions/2025_12_25_1039-7df29de0f6be_add_credit_pool.py

@@ -8,6 +8,7 @@ Create Date: 2025-12-25 10:39:15.139304
 from alembic import op
 import models as models
 import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = '7df29de0f6be'
@@ -19,7 +20,7 @@ depends_on = None
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
     op.create_table('tenant_credit_pools',
-    sa.Column('id', models.types.StringUUID(), nullable=False),
+    sa.Column('id', models.types.StringUUID(), server_default=sa.text('uuid_generate_v4()'), nullable=False),
     sa.Column('tenant_id', models.types.StringUUID(), nullable=False),
     sa.Column('pool_type', sa.String(length=40), server_default='trial', nullable=False),
     sa.Column('quota_limit', sa.BigInteger(), nullable=False),