chore(version): bump version to 1.7.2 (#23740)

Co-authored-by: crazywoola <100913391+crazywoola@users.noreply.github.com>

.github/ISSUE_TEMPLATE/chore.yaml

@@ -0,0 +1,44 @@
+name: "✨ Refactor"
+description: Refactor existing code for improved readability and maintainability.
+title: "[Chore/Refactor] "
+labels:
+  - refactor
+body:
+  - type: checkboxes
+    attributes:
+      label: Self Checks
+      description: "To make sure we get to you in time, please check the following :)"
+      options:
+        - label: I have read the [Contributing Guide](https://github.com/langgenius/dify/blob/main/CONTRIBUTING.md) and [Language Policy](https://github.com/langgenius/dify/issues/1542).
+          required: true
+        - label: This is only for refactoring, if you would like to ask a question, please head to [Discussions](https://github.com/langgenius/dify/discussions/categories/general).
+          required: true
+        - label: I have searched for existing issues [search for existing issues](https://github.com/langgenius/dify/issues), including closed ones.
+          required: true
+        - label: I confirm that I am using English to submit this report, otherwise it will be closed.
+          required: true
+        - label: 【中文用户 & Non English User】请使用英语提交，否则会被关闭 ：）
+          required: true
+        - label: "Please do not modify this template :) and fill in all the required fields."
+          required: true
+  - type: textarea
+    id: description
+    attributes:
+      label: Description
+      placeholder: "Describe the refactor you are proposing."
+    validations:
+      required: true
+  - type: textarea
+    id: motivation
+    attributes:
+      label: Motivation
+      placeholder: "Explain why this refactor is necessary."
+    validations:
+      required: false
+  - type: textarea
+    id: additional-context
+    attributes:
+      label: Additional Context
+      placeholder: "Add any other context or screenshots about the request here."
+    validations:
+      required: false

.github/workflows/api-tests.yml

@@ -99,3 +99,6 @@ jobs:
 
       - name: Run Tool
         run: uv run --project api bash dev/pytest/pytest_tools.sh
+
+      - name: Run TestContainers
+        run: uv run --project api bash dev/pytest/pytest_testcontainers.sh

.github/workflows/autofix.yml

@@ -9,6 +9,7 @@ permissions:
 
 jobs:
   autofix:
+    if: github.repository == 'langgenius/dify'
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.github/workflows/build-push.yml

@@ -7,6 +7,7 @@ on:
       - "deploy/dev"
       - "deploy/enterprise"
       - "build/**"
+      - "release/e-*"
     tags:
       - "*"
 

.github/workflows/translate-i18n-base-on-english.yml

@@ -1,13 +1,18 @@
 name: Check i18n Files and Create PR
 
 on:
-  pull_request:
-    types: [closed]
+  push:
     branches: [main]
+    paths:
+      - 'web/i18n/en-US/*.ts'
+
+permissions:
+  contents: write
+  pull-requests: write
 
 jobs:
   check-and-update:
-    if: github.event.pull_request.merged == true
+    if: github.repository == 'langgenius/dify'
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -15,8 +20,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 2 # last 2 commits
-          persist-credentials: false
+          fetch-depth: 2
+          token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Check for file changes in i18n/en-US
         id: check_files
@@ -27,6 +32,13 @@ jobs:
           echo "Changed files: $changed_files"
           if [ -n "$changed_files" ]; then
             echo "FILES_CHANGED=true" >> $GITHUB_ENV
+            file_args=""
+            for file in $changed_files; do
+              filename=$(basename "$file" .ts)
+              file_args="$file_args --file=$filename"
+            done
+            echo "FILE_ARGS=$file_args" >> $GITHUB_ENV
+            echo "File arguments: $file_args"
           else
             echo "FILES_CHANGED=false" >> $GITHUB_ENV
           fi
@@ -49,14 +61,15 @@ jobs:
         if: env.FILES_CHANGED == 'true'
         run: pnpm install --frozen-lockfile
 
-      - name: Run npm script
+      - name: Generate i18n translations
         if: env.FILES_CHANGED == 'true'
-        run: pnpm run auto-gen-i18n
+        run: pnpm run auto-gen-i18n ${{ env.FILE_ARGS }}
 
       - name: Create Pull Request
         if: env.FILES_CHANGED == 'true'
         uses: peter-evans/create-pull-request@v6
         with:
+          token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: Update i18n files based on en-US changes
           title: 'chore: translate i18n files'
           body: This PR was automatically created to update i18n files based on changes in en-US locale.

.gitignore

@@ -215,3 +215,4 @@ mise.toml
 # AI Assistant
 .roo/
 api/.env.backup
+/clickzetta

api/Dockerfile

@@ -19,7 +19,7 @@ RUN apt-get update \
 
 # Install Python dependencies
 COPY pyproject.toml uv.lock ./
-RUN uv sync --locked
+RUN uv sync --locked --no-dev
 
 # production stage
 FROM base AS production

api/commands.py

@@ -5,10 +5,11 @@ import secrets
 from typing import Any, Optional
 
 import click
+import sqlalchemy as sa
 from flask import current_app
 from pydantic import TypeAdapter
 from sqlalchemy import select
-from werkzeug.exceptions import NotFound
+from sqlalchemy.exc import SQLAlchemyError
 
 from configs import dify_config
 from constants.languages import languages
@@ -180,8 +181,8 @@ def migrate_annotation_vector_database():
             )
             if not apps:
                 break
-        except NotFound:
-            break
+        except SQLAlchemyError:
+            raise
 
         page += 1
         for app in apps:
@@ -307,8 +308,8 @@ def migrate_knowledge_vector_database():
             )
 
             datasets = db.paginate(select=stmt, page=page, per_page=50, max_per_page=50, error_out=False)
-        except NotFound:
-            break
+        except SQLAlchemyError:
+            raise
 
         page += 1
         for dataset in datasets:
@@ -457,7 +458,7 @@ def convert_to_agent_apps():
         """
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query))
+            rs = conn.execute(sa.text(sql_query))
 
             apps = []
             for i in rs:
@@ -560,8 +561,8 @@ def old_metadata_migration():
                 .order_by(DatasetDocument.created_at.desc())
             )
             documents = db.paginate(select=stmt, page=page, per_page=50, max_per_page=50, error_out=False)
-        except NotFound:
-            break
+        except SQLAlchemyError:
+            raise
         if not documents:
             break
         for document in documents:
@@ -702,7 +703,7 @@ def fix_app_site_missing():
         sql = """select apps.id as id from apps left join sites on sites.app_id=apps.id
 where sites.id is null limit 1000"""
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql))
+            rs = conn.execute(sa.text(sql))
 
             processed_count = 0
             for i in rs:
@@ -916,7 +917,7 @@ def clear_orphaned_file_records(force: bool):
         )
         orphaned_message_files = []
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(query))
+            rs = conn.execute(sa.text(query))
             for i in rs:
                 orphaned_message_files.append({"id": str(i[0]), "message_id": str(i[1])})
 
@@ -937,7 +938,7 @@ def clear_orphaned_file_records(force: bool):
             click.echo(click.style("- Deleting orphaned message_files records", fg="white"))
             query = "DELETE FROM message_files WHERE id IN :ids"
             with db.engine.begin() as conn:
-                conn.execute(db.text(query), {"ids": tuple([record["id"] for record in orphaned_message_files])})
+                conn.execute(sa.text(query), {"ids": tuple([record["id"] for record in orphaned_message_files])})
             click.echo(
                 click.style(f"Removed {len(orphaned_message_files)} orphaned message_files records.", fg="green")
             )
@@ -954,7 +955,7 @@ def clear_orphaned_file_records(force: bool):
             click.echo(click.style(f"- Listing file records in table {files_table['table']}", fg="white"))
             query = f"SELECT {files_table['id_column']}, {files_table['key_column']} FROM {files_table['table']}"
             with db.engine.begin() as conn:
-                rs = conn.execute(db.text(query))
+                rs = conn.execute(sa.text(query))
             for i in rs:
                 all_files_in_tables.append({"table": files_table["table"], "id": str(i[0]), "key": i[1]})
         click.echo(click.style(f"Found {len(all_files_in_tables)} files in tables.", fg="white"))
@@ -974,7 +975,7 @@ def clear_orphaned_file_records(force: bool):
                     f"SELECT {ids_table['column']} FROM {ids_table['table']} WHERE {ids_table['column']} IS NOT NULL"
                 )
                 with db.engine.begin() as conn:
-                    rs = conn.execute(db.text(query))
+                    rs = conn.execute(sa.text(query))
                 for i in rs:
                     all_ids_in_tables.append({"table": ids_table["table"], "id": str(i[0])})
             elif ids_table["type"] == "text":
@@ -989,7 +990,7 @@ def clear_orphaned_file_records(force: bool):
                     f"FROM {ids_table['table']}"
                 )
                 with db.engine.begin() as conn:
-                    rs = conn.execute(db.text(query))
+                    rs = conn.execute(sa.text(query))
                 for i in rs:
                     for j in i[0]:
                         all_ids_in_tables.append({"table": ids_table["table"], "id": j})
@@ -1008,7 +1009,7 @@ def clear_orphaned_file_records(force: bool):
                     f"FROM {ids_table['table']}"
                 )
                 with db.engine.begin() as conn:
-                    rs = conn.execute(db.text(query))
+                    rs = conn.execute(sa.text(query))
                 for i in rs:
                     for j in i[0]:
                         all_ids_in_tables.append({"table": ids_table["table"], "id": j})
@@ -1037,7 +1038,7 @@ def clear_orphaned_file_records(force: bool):
             click.echo(click.style(f"- Deleting orphaned file records in table {files_table['table']}", fg="white"))
             query = f"DELETE FROM {files_table['table']} WHERE {files_table['id_column']} IN :ids"
             with db.engine.begin() as conn:
-                conn.execute(db.text(query), {"ids": tuple(orphaned_files)})
+                conn.execute(sa.text(query), {"ids": tuple(orphaned_files)})
     except Exception as e:
         click.echo(click.style(f"Error deleting orphaned file records: {str(e)}", fg="red"))
         return
@@ -1107,7 +1108,7 @@ def remove_orphaned_files_on_storage(force: bool):
             click.echo(click.style(f"- Listing files from table {files_table['table']}", fg="white"))
             query = f"SELECT {files_table['key_column']} FROM {files_table['table']}"
             with db.engine.begin() as conn:
-                rs = conn.execute(db.text(query))
+                rs = conn.execute(sa.text(query))
             for i in rs:
                 all_files_in_tables.append(str(i[0]))
         click.echo(click.style(f"Found {len(all_files_in_tables)} files in tables.", fg="white"))

api/configs/feature/__init__.py

@@ -330,17 +330,17 @@ class HttpConfig(BaseSettings):
     def WEB_API_CORS_ALLOW_ORIGINS(self) -> list[str]:
         return self.inner_WEB_API_CORS_ALLOW_ORIGINS.split(",")
 
-    HTTP_REQUEST_MAX_CONNECT_TIMEOUT: Annotated[
-        PositiveInt, Field(ge=10, description="Maximum connection timeout in seconds for HTTP requests")
-    ] = 10
+    HTTP_REQUEST_MAX_CONNECT_TIMEOUT: int = Field(
+        ge=1, description="Maximum connection timeout in seconds for HTTP requests", default=10
+    )
 
-    HTTP_REQUEST_MAX_READ_TIMEOUT: Annotated[
-        PositiveInt, Field(ge=60, description="Maximum read timeout in seconds for HTTP requests")
-    ] = 60
+    HTTP_REQUEST_MAX_READ_TIMEOUT: int = Field(
+        ge=1, description="Maximum read timeout in seconds for HTTP requests", default=60
+    )
 
-    HTTP_REQUEST_MAX_WRITE_TIMEOUT: Annotated[
-        PositiveInt, Field(ge=10, description="Maximum write timeout in seconds for HTTP requests")
-    ] = 20
+    HTTP_REQUEST_MAX_WRITE_TIMEOUT: int = Field(
+        ge=1, description="Maximum write timeout in seconds for HTTP requests", default=20
+    )
 
     HTTP_REQUEST_NODE_MAX_BINARY_SIZE: PositiveInt = Field(
         description="Maximum allowed size in bytes for binary data in HTTP requests",

api/configs/middleware/__init__.py

@@ -10,6 +10,7 @@ from .storage.aliyun_oss_storage_config import AliyunOSSStorageConfig
 from .storage.amazon_s3_storage_config import S3StorageConfig
 from .storage.azure_blob_storage_config import AzureBlobStorageConfig
 from .storage.baidu_obs_storage_config import BaiduOBSStorageConfig
+from .storage.clickzetta_volume_storage_config import ClickZettaVolumeStorageConfig
 from .storage.google_cloud_storage_config import GoogleCloudStorageConfig
 from .storage.huawei_obs_storage_config import HuaweiCloudOBSStorageConfig
 from .storage.oci_storage_config import OCIStorageConfig
@@ -20,6 +21,7 @@ from .storage.volcengine_tos_storage_config import VolcengineTOSStorageConfig
 from .vdb.analyticdb_config import AnalyticdbConfig
 from .vdb.baidu_vector_config import BaiduVectorDBConfig
 from .vdb.chroma_config import ChromaConfig
+from .vdb.clickzetta_config import ClickzettaConfig
 from .vdb.couchbase_config import CouchbaseConfig
 from .vdb.elasticsearch_config import ElasticsearchConfig
 from .vdb.huawei_cloud_config import HuaweiCloudConfig
@@ -52,6 +54,7 @@ class StorageConfig(BaseSettings):
         "aliyun-oss",
         "azure-blob",
         "baidu-obs",
+        "clickzetta-volume",
         "google-storage",
         "huawei-obs",
         "oci-storage",
@@ -61,8 +64,9 @@ class StorageConfig(BaseSettings):
         "local",
     ] = Field(
         description="Type of storage to use."
-        " Options: 'opendal', '(deprecated) local', 's3', 'aliyun-oss', 'azure-blob', 'baidu-obs', 'google-storage', "
-        "'huawei-obs', 'oci-storage', 'tencent-cos', 'volcengine-tos', 'supabase'. Default is 'opendal'.",
+        " Options: 'opendal', '(deprecated) local', 's3', 'aliyun-oss', 'azure-blob', 'baidu-obs', "
+        "'clickzetta-volume', 'google-storage', 'huawei-obs', 'oci-storage', 'tencent-cos', "
+        "'volcengine-tos', 'supabase'. Default is 'opendal'.",
         default="opendal",
     )
 
@@ -140,7 +144,8 @@ class DatabaseConfig(BaseSettings):
         default="postgresql",
     )
 
-    @computed_field
+    @computed_field  # type: ignore[misc]
+    @property
     def SQLALCHEMY_DATABASE_URI(self) -> str:
         db_extras = (
             f"{self.DB_EXTRAS}&client_encoding={self.DB_CHARSET}" if self.DB_CHARSET else self.DB_EXTRAS
@@ -303,6 +308,7 @@ class MiddlewareConfig(
     AliyunOSSStorageConfig,
     AzureBlobStorageConfig,
     BaiduOBSStorageConfig,
+    ClickZettaVolumeStorageConfig,
     GoogleCloudStorageConfig,
     HuaweiCloudOBSStorageConfig,
     OCIStorageConfig,
@@ -315,6 +321,7 @@ class MiddlewareConfig(
     VectorStoreConfig,
     AnalyticdbConfig,
     ChromaConfig,
+    ClickzettaConfig,
     HuaweiCloudConfig,
     MilvusConfig,
     MyScaleConfig,

api/configs/middleware/storage/clickzetta_volume_storage_config.py

@@ -0,0 +1,65 @@
+"""ClickZetta Volume Storage Configuration"""
+
+from typing import Optional
+
+from pydantic import Field
+from pydantic_settings import BaseSettings
+
+
+class ClickZettaVolumeStorageConfig(BaseSettings):
+    """Configuration for ClickZetta Volume storage."""
+
+    CLICKZETTA_VOLUME_USERNAME: Optional[str] = Field(
+        description="Username for ClickZetta Volume authentication",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_PASSWORD: Optional[str] = Field(
+        description="Password for ClickZetta Volume authentication",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_INSTANCE: Optional[str] = Field(
+        description="ClickZetta instance identifier",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_SERVICE: str = Field(
+        description="ClickZetta service endpoint",
+        default="api.clickzetta.com",
+    )
+
+    CLICKZETTA_VOLUME_WORKSPACE: str = Field(
+        description="ClickZetta workspace name",
+        default="quick_start",
+    )
+
+    CLICKZETTA_VOLUME_VCLUSTER: str = Field(
+        description="ClickZetta virtual cluster name",
+        default="default_ap",
+    )
+
+    CLICKZETTA_VOLUME_SCHEMA: str = Field(
+        description="ClickZetta schema name",
+        default="dify",
+    )
+
+    CLICKZETTA_VOLUME_TYPE: str = Field(
+        description="ClickZetta volume type (table|user|external)",
+        default="user",
+    )
+
+    CLICKZETTA_VOLUME_NAME: Optional[str] = Field(
+        description="ClickZetta volume name for external volumes",
+        default=None,
+    )
+
+    CLICKZETTA_VOLUME_TABLE_PREFIX: str = Field(
+        description="Prefix for ClickZetta volume table names",
+        default="dataset_",
+    )
+
+    CLICKZETTA_VOLUME_DIFY_PREFIX: str = Field(
+        description="Directory prefix for User Volume to organize Dify files",
+        default="dify_km",
+    )

api/configs/middleware/vdb/clickzetta_config.py

@@ -0,0 +1,69 @@
+from typing import Optional
+
+from pydantic import BaseModel, Field
+
+
+class ClickzettaConfig(BaseModel):
+    """
+    Clickzetta Lakehouse vector database configuration
+    """
+
+    CLICKZETTA_USERNAME: Optional[str] = Field(
+        description="Username for authenticating with Clickzetta Lakehouse",
+        default=None,
+    )
+
+    CLICKZETTA_PASSWORD: Optional[str] = Field(
+        description="Password for authenticating with Clickzetta Lakehouse",
+        default=None,
+    )
+
+    CLICKZETTA_INSTANCE: Optional[str] = Field(
+        description="Clickzetta Lakehouse instance ID",
+        default=None,
+    )
+
+    CLICKZETTA_SERVICE: Optional[str] = Field(
+        description="Clickzetta API service endpoint (e.g., 'api.clickzetta.com')",
+        default="api.clickzetta.com",
+    )
+
+    CLICKZETTA_WORKSPACE: Optional[str] = Field(
+        description="Clickzetta workspace name",
+        default="default",
+    )
+
+    CLICKZETTA_VCLUSTER: Optional[str] = Field(
+        description="Clickzetta virtual cluster name",
+        default="default_ap",
+    )
+
+    CLICKZETTA_SCHEMA: Optional[str] = Field(
+        description="Database schema name in Clickzetta",
+        default="public",
+    )
+
+    CLICKZETTA_BATCH_SIZE: Optional[int] = Field(
+        description="Batch size for bulk insert operations",
+        default=100,
+    )
+
+    CLICKZETTA_ENABLE_INVERTED_INDEX: Optional[bool] = Field(
+        description="Enable inverted index for full-text search capabilities",
+        default=True,
+    )
+
+    CLICKZETTA_ANALYZER_TYPE: Optional[str] = Field(
+        description="Analyzer type for full-text search: keyword, english, chinese, unicode",
+        default="chinese",
+    )
+
+    CLICKZETTA_ANALYZER_MODE: Optional[str] = Field(
+        description="Analyzer mode for tokenization: max_word (fine-grained) or smart (intelligent)",
+        default="smart",
+    )
+
+    CLICKZETTA_VECTOR_DISTANCE_FUNCTION: Optional[str] = Field(
+        description="Distance function for vector similarity: l2_distance or cosine_distance",
+        default="cosine_distance",
+    )

api/constants/__init__.py

@@ -9,10 +9,10 @@ DEFAULT_FILE_NUMBER_LIMITS = 3
 IMAGE_EXTENSIONS = ["jpg", "jpeg", "png", "webp", "gif", "svg"]
 IMAGE_EXTENSIONS.extend([ext.upper() for ext in IMAGE_EXTENSIONS])
 
-VIDEO_EXTENSIONS = ["mp4", "mov", "mpeg", "mpga"]
+VIDEO_EXTENSIONS = ["mp4", "mov", "mpeg", "webm"]
 VIDEO_EXTENSIONS.extend([ext.upper() for ext in VIDEO_EXTENSIONS])
 
-AUDIO_EXTENSIONS = ["mp3", "m4a", "wav", "webm", "amr"]
+AUDIO_EXTENSIONS = ["mp3", "m4a", "wav", "amr", "mpga"]
 AUDIO_EXTENSIONS.extend([ext.upper() for ext in AUDIO_EXTENSIONS])
 
 

api/controllers/console/__init__.py

@@ -84,6 +84,7 @@ from .datasets import (
     external,
     hit_testing,
     metadata,
+    upload_file,
     website,
 )
 

api/controllers/console/app/annotation.py

@@ -225,14 +225,15 @@ class AnnotationBatchImportApi(Resource):
             raise Forbidden()
 
         app_id = str(app_id)
-        # get file from request
-        file = request.files["file"]
         # check file
         if "file" not in request.files:
             raise NoFileUploadedError()
 
         if len(request.files) > 1:
             raise TooManyFilesError()
+
+        # get file from request
+        file = request.files["file"]
         # check file type
         if not file.filename or not file.filename.lower().endswith(".csv"):
             raise ValueError("Invalid file type. Only CSV files are allowed")

api/controllers/console/app/app.py

@@ -28,6 +28,12 @@ from services.feature_service import FeatureService
 ALLOW_CREATE_APP_MODES = ["chat", "agent-chat", "advanced-chat", "workflow", "completion"]
 
 
+def _validate_description_length(description):
+    if description and len(description) > 400:
+        raise ValueError("Description cannot exceed 400 characters.")
+    return description
+
+
 class AppListApi(Resource):
     @setup_required
     @login_required
@@ -94,7 +100,7 @@ class AppListApi(Resource):
         """Create app"""
         parser = reqparse.RequestParser()
         parser.add_argument("name", type=str, required=True, location="json")
-        parser.add_argument("description", type=str, location="json")
+        parser.add_argument("description", type=_validate_description_length, location="json")
         parser.add_argument("mode", type=str, choices=ALLOW_CREATE_APP_MODES, location="json")
         parser.add_argument("icon_type", type=str, location="json")
         parser.add_argument("icon", type=str, location="json")
@@ -146,7 +152,7 @@ class AppApi(Resource):
 
         parser = reqparse.RequestParser()
         parser.add_argument("name", type=str, required=True, nullable=False, location="json")
-        parser.add_argument("description", type=str, location="json")
+        parser.add_argument("description", type=_validate_description_length, location="json")
         parser.add_argument("icon_type", type=str, location="json")
         parser.add_argument("icon", type=str, location="json")
         parser.add_argument("icon_background", type=str, location="json")
@@ -189,7 +195,7 @@ class AppCopyApi(Resource):
 
         parser = reqparse.RequestParser()
         parser.add_argument("name", type=str, location="json")
-        parser.add_argument("description", type=str, location="json")
+        parser.add_argument("description", type=_validate_description_length, location="json")
         parser.add_argument("icon_type", type=str, location="json")
         parser.add_argument("icon", type=str, location="json")
         parser.add_argument("icon_background", type=str, location="json")

api/controllers/console/app/statistic.py

@@ -67,7 +67,7 @@ WHERE
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append({"date": str(i.date), "message_count": i.message_count})
 
@@ -176,7 +176,7 @@ WHERE
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append({"date": str(i.date), "terminal_count": i.terminal_count})
 
@@ -234,7 +234,7 @@ WHERE
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append(
                     {"date": str(i.date), "token_count": i.token_count, "total_price": i.total_price, "currency": "USD"}
@@ -310,7 +310,7 @@ ORDER BY
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append(
                     {"date": str(i.date), "interactions": float(i.interactions.quantize(Decimal("0.01")))}
@@ -373,7 +373,7 @@ WHERE
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append(
                     {
@@ -435,7 +435,7 @@ WHERE
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append({"date": str(i.date), "latency": round(i.latency * 1000, 4)})
 
@@ -495,7 +495,7 @@ WHERE
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append({"date": str(i.date), "tps": round(i.tokens_per_second, 4)})
 

api/controllers/console/app/workflow_statistic.py

@@ -2,6 +2,7 @@ from datetime import datetime
 from decimal import Decimal
 
 import pytz
+import sqlalchemy as sa
 from flask import jsonify
 from flask_login import current_user
 from flask_restful import Resource, reqparse
@@ -71,7 +72,7 @@ WHERE
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append({"date": str(i.date), "runs": i.runs})
 
@@ -133,7 +134,7 @@ WHERE
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append({"date": str(i.date), "terminal_count": i.terminal_count})
 
@@ -195,7 +196,7 @@ WHERE
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append(
                     {
@@ -277,7 +278,7 @@ GROUP BY
         response_data = []
 
         with db.engine.begin() as conn:
-            rs = conn.execute(db.text(sql_query), arg_dict)
+            rs = conn.execute(sa.text(sql_query), arg_dict)
             for i in rs:
                 response_data.append(
                     {"date": str(i.date), "interactions": float(i.interactions.quantize(Decimal("0.01")))}

api/controllers/console/datasets/datasets.py

@@ -41,7 +41,7 @@ def _validate_name(name):
 
 
 def _validate_description_length(description):
-    if len(description) > 400:
+    if description and len(description) > 400:
         raise ValueError("Description cannot exceed 400 characters.")
     return description
 
@@ -113,7 +113,7 @@ class DatasetListApi(Resource):
         )
         parser.add_argument(
             "description",
-            type=str,
+            type=_validate_description_length,
             nullable=True,
             required=False,
             default="",
@@ -683,6 +683,7 @@ class DatasetRetrievalSettingApi(Resource):
                 | VectorType.HUAWEI_CLOUD
                 | VectorType.TENCENT
                 | VectorType.MATRIXONE
+                | VectorType.CLICKZETTA
             ):
                 return {
                     "retrieval_method": [
@@ -731,6 +732,7 @@ class DatasetRetrievalSettingMockApi(Resource):
                 | VectorType.TENCENT
                 | VectorType.HUAWEI_CLOUD
                 | VectorType.MATRIXONE
+                | VectorType.CLICKZETTA
             ):
                 return {
                     "retrieval_method": [

api/controllers/console/datasets/datasets_document.py

@@ -642,7 +642,7 @@ class DocumentIndexingStatusApi(DocumentResource):
         return marshal(document_dict, document_status_fields)
 
 
-class DocumentDetailApi(DocumentResource):
+class DocumentApi(DocumentResource):
     METADATA_CHOICES = {"all", "only", "without"}
 
     @setup_required
@@ -730,6 +730,28 @@ class DocumentDetailApi(DocumentResource):
 
         return response, 200
 
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @cloud_edition_billing_rate_limit_check("knowledge")
+    def delete(self, dataset_id, document_id):
+        dataset_id = str(dataset_id)
+        document_id = str(document_id)
+        dataset = DatasetService.get_dataset(dataset_id)
+        if dataset is None:
+            raise NotFound("Dataset not found.")
+        # check user's model setting
+        DatasetService.check_dataset_model_setting(dataset)
+
+        document = self.get_document(dataset_id, document_id)
+
+        try:
+            DocumentService.delete_document(document)
+        except services.errors.document.DocumentIndexingError:
+            raise DocumentIndexingError("Cannot delete document during indexing.")
+
+        return {"result": "success"}, 204
+
 
 class DocumentProcessingApi(DocumentResource):
     @setup_required
@@ -768,30 +790,6 @@ class DocumentProcessingApi(DocumentResource):
         return {"result": "success"}, 200
 
 
-class DocumentDeleteApi(DocumentResource):
-    @setup_required
-    @login_required
-    @account_initialization_required
-    @cloud_edition_billing_rate_limit_check("knowledge")
-    def delete(self, dataset_id, document_id):
-        dataset_id = str(dataset_id)
-        document_id = str(document_id)
-        dataset = DatasetService.get_dataset(dataset_id)
-        if dataset is None:
-            raise NotFound("Dataset not found.")
-        # check user's model setting
-        DatasetService.check_dataset_model_setting(dataset)
-
-        document = self.get_document(dataset_id, document_id)
-
-        try:
-            DocumentService.delete_document(document)
-        except services.errors.document.DocumentIndexingError:
-            raise DocumentIndexingError("Cannot delete document during indexing.")
-
-        return {"result": "success"}, 204
-
-
 class DocumentMetadataApi(DocumentResource):
     @setup_required
     @login_required
@@ -1037,11 +1035,10 @@ api.add_resource(
 api.add_resource(DocumentBatchIndexingEstimateApi, "/datasets/<uuid:dataset_id>/batch/<string:batch>/indexing-estimate")
 api.add_resource(DocumentBatchIndexingStatusApi, "/datasets/<uuid:dataset_id>/batch/<string:batch>/indexing-status")
 api.add_resource(DocumentIndexingStatusApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/indexing-status")
-api.add_resource(DocumentDetailApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
+api.add_resource(DocumentApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
 api.add_resource(
     DocumentProcessingApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/<string:action>"
 )
-api.add_resource(DocumentDeleteApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
 api.add_resource(DocumentMetadataApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/metadata")
 api.add_resource(DocumentStatusApi, "/datasets/<uuid:dataset_id>/documents/status/<string:action>/batch")
 api.add_resource(DocumentPauseApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/processing/pause")

api/controllers/console/datasets/upload_file.py

@@ -0,0 +1,62 @@
+from flask_login import current_user
+from flask_restful import Resource
+from werkzeug.exceptions import NotFound
+
+from controllers.console import api
+from controllers.console.wraps import (
+    account_initialization_required,
+    setup_required,
+)
+from core.file import helpers as file_helpers
+from extensions.ext_database import db
+from models.dataset import Dataset
+from models.model import UploadFile
+from services.dataset_service import DocumentService
+
+
+class UploadFileApi(Resource):
+    @setup_required
+    @account_initialization_required
+    def get(self, dataset_id, document_id):
+        """Get upload file."""
+        # check dataset
+        dataset_id = str(dataset_id)
+        dataset = (
+            db.session.query(Dataset)
+            .filter(Dataset.tenant_id == current_user.current_tenant_id, Dataset.id == dataset_id)
+            .first()
+        )
+        if not dataset:
+            raise NotFound("Dataset not found.")
+        # check document
+        document_id = str(document_id)
+        document = DocumentService.get_document(dataset.id, document_id)
+        if not document:
+            raise NotFound("Document not found.")
+        # check upload file
+        if document.data_source_type != "upload_file":
+            raise ValueError(f"Document data source type ({document.data_source_type}) is not upload_file.")
+        data_source_info = document.data_source_info_dict
+        if data_source_info and "upload_file_id" in data_source_info:
+            file_id = data_source_info["upload_file_id"]
+            upload_file = db.session.query(UploadFile).filter(UploadFile.id == file_id).first()
+            if not upload_file:
+                raise NotFound("UploadFile not found.")
+        else:
+            raise ValueError("Upload file id not found in document data source info.")
+
+        url = file_helpers.get_signed_file_url(upload_file_id=upload_file.id)
+        return {
+            "id": upload_file.id,
+            "name": upload_file.name,
+            "size": upload_file.size,
+            "extension": upload_file.extension,
+            "url": url,
+            "download_url": f"{url}&as_attachment=true",
+            "mime_type": upload_file.mime_type,
+            "created_by": upload_file.created_by,
+            "created_at": upload_file.created_at.timestamp(),
+        }, 200
+
+
+api.add_resource(UploadFileApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/upload-file")

api/controllers/console/error.py

@@ -127,7 +127,7 @@ class EducationActivateLimitError(BaseHTTPException):
     code = 429
 
 
-class CompilanceRateLimitError(BaseHTTPException):
-    error_code = "compilance_rate_limit"
+class ComplianceRateLimitError(BaseHTTPException):
+    error_code = "compliance_rate_limit"
     description = "Rate limit exceeded for downloading compliance report."
     code = 429

api/controllers/console/explore/installed_app.py

@@ -58,21 +58,38 @@ class InstalledAppsListApi(Resource):
         # filter out apps that user doesn't have access to
         if FeatureService.get_system_features().webapp_auth.enabled:
             user_id = current_user.id
-            res = []
             app_ids = [installed_app["app"].id for installed_app in installed_app_list]
             webapp_settings = EnterpriseService.WebAppAuth.batch_get_app_access_mode_by_id(app_ids)
+
+            # Pre-filter out apps without setting or with sso_verified
+            filtered_installed_apps = []
+            app_id_to_app_code = {}
+
             for installed_app in installed_app_list:
-                webapp_setting = webapp_settings.get(installed_app["app"].id)
-                if not webapp_setting:
+                app_id = installed_app["app"].id
+                webapp_setting = webapp_settings.get(app_id)
+                if not webapp_setting or webapp_setting.access_mode == "sso_verified":
                     continue
-                if webapp_setting.access_mode == "sso_verified":
-                    continue
-                app_code = AppService.get_app_code_by_id(str(installed_app["app"].id))
-                if EnterpriseService.WebAppAuth.is_user_allowed_to_access_webapp(
-                    user_id=user_id,
-                    app_code=app_code,
-                ):
+                app_code = AppService.get_app_code_by_id(str(app_id))
+                app_id_to_app_code[app_id] = app_code
+                filtered_installed_apps.append(installed_app)
+
+            app_codes = list(app_id_to_app_code.values())
+
+            # Batch permission check
+            permissions = EnterpriseService.WebAppAuth.batch_is_user_allowed_to_access_webapps(
+                user_id=user_id,
+                app_codes=app_codes,
+            )
+
+            # Keep only allowed apps
+            res = []
+            for installed_app in filtered_installed_apps:
+                app_id = installed_app["app"].id
+                app_code = app_id_to_app_code[app_id]
+                if permissions.get(app_code):
                     res.append(installed_app)
+
             installed_app_list = res
             logger.debug("installed_app_list: %s, user_id: %s", installed_app_list, user_id)
 

api/controllers/console/files.py

@@ -49,7 +49,6 @@ class FileApi(Resource):
     @marshal_with(file_fields)
     @cloud_edition_billing_resource_check("documents")
     def post(self):
-        file = request.files["file"]
         source_str = request.form.get("source")
         source: Literal["datasets"] | None = "datasets" if source_str == "datasets" else None
 
@@ -58,6 +57,7 @@ class FileApi(Resource):
 
         if len(request.files) > 1:
             raise TooManyFilesError()
+        file = request.files["file"]
 
         if not file.filename:
             raise FilenameNotExistsError

api/controllers/console/workspace/workspace.py

@@ -191,9 +191,6 @@ class WebappLogoWorkspaceApi(Resource):
     @account_initialization_required
     @cloud_edition_billing_resource_check("workspace_custom")
     def post(self):
-        # get file from request
-        file = request.files["file"]
-
         # check file
         if "file" not in request.files:
             raise NoFileUploadedError()
@@ -201,6 +198,8 @@ class WebappLogoWorkspaceApi(Resource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
+        # get file from request
+        file = request.files["file"]
         if not file.filename:
             raise FilenameNotExistsError
 

api/controllers/service_api/__init__.py

@@ -6,6 +6,6 @@ bp = Blueprint("service_api", __name__, url_prefix="/v1")
 api = ExternalApi(bp)
 
 from . import index
-from .app import annotation, app, audio, completion, conversation, file, message, site, workflow
+from .app import annotation, app, audio, completion, conversation, file, file_preview, message, site, workflow
 from .dataset import dataset, document, hit_testing, metadata, segment, upload_file
 from .workspace import models

api/controllers/service_api/app/completion.py

@@ -2,7 +2,7 @@ import logging
 
 from flask import request
 from flask_restful import Resource, reqparse
-from werkzeug.exceptions import InternalServerError, NotFound
+from werkzeug.exceptions import BadRequest, InternalServerError, NotFound
 
 import services
 from controllers.service_api import api
@@ -30,6 +30,7 @@ from libs import helper
 from libs.helper import uuid_value
 from models.model import App, AppMode, EndUser
 from services.app_generate_service import AppGenerateService
+from services.errors.app import IsDraftWorkflowError, WorkflowIdFormatError, WorkflowNotFoundError
 from services.errors.llm import InvokeRateLimitError
 
 
@@ -113,7 +114,7 @@ class ChatApi(Resource):
         parser.add_argument("conversation_id", type=uuid_value, location="json")
         parser.add_argument("retriever_from", type=str, required=False, default="dev", location="json")
         parser.add_argument("auto_generate_name", type=bool, required=False, default=True, location="json")
-
+        parser.add_argument("workflow_id", type=str, required=False, location="json")
         args = parser.parse_args()
 
         external_trace_id = get_external_trace_id(request)
@@ -128,6 +129,12 @@ class ChatApi(Resource):
             )
 
             return helper.compact_generate_response(response)
+        except WorkflowNotFoundError as ex:
+            raise NotFound(str(ex))
+        except IsDraftWorkflowError as ex:
+            raise BadRequest(str(ex))
+        except WorkflowIdFormatError as ex:
+            raise BadRequest(str(ex))
         except services.errors.conversation.ConversationNotExistsError:
             raise NotFound("Conversation Not Exists.")
         except services.errors.conversation.ConversationCompletedError:

api/controllers/service_api/app/error.py

@@ -107,3 +107,15 @@ class UnsupportedFileTypeError(BaseHTTPException):
     error_code = "unsupported_file_type"
     description = "File type not allowed."
     code = 415
+
+
+class FileNotFoundError(BaseHTTPException):
+    error_code = "file_not_found"
+    description = "The requested file was not found."
+    code = 404
+
+
+class FileAccessDeniedError(BaseHTTPException):
+    error_code = "file_access_denied"
+    description = "Access to the requested file is denied."
+    code = 403

api/controllers/service_api/app/file.py

@@ -20,18 +20,17 @@ class FileApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.FORM))
     @marshal_with(file_fields)
     def post(self, app_model: App, end_user: EndUser):
-        file = request.files["file"]
-
         # check file
         if "file" not in request.files:
             raise NoFileUploadedError()
 
-        if not file.mimetype:
-            raise UnsupportedFileTypeError()
-
         if len(request.files) > 1:
             raise TooManyFilesError()
 
+        file = request.files["file"]
+        if not file.mimetype:
+            raise UnsupportedFileTypeError()
+
         if not file.filename:
             raise FilenameNotExistsError
 

api/controllers/service_api/app/file_preview.py

@@ -0,0 +1,186 @@
+import logging
+from urllib.parse import quote
+
+from flask import Response
+from flask_restful import Resource, reqparse
+
+from controllers.service_api import api
+from controllers.service_api.app.error import (
+    FileAccessDeniedError,
+    FileNotFoundError,
+)
+from controllers.service_api.wraps import FetchUserArg, WhereisUserArg, validate_app_token
+from extensions.ext_database import db
+from extensions.ext_storage import storage
+from models.model import App, EndUser, Message, MessageFile, UploadFile
+
+logger = logging.getLogger(__name__)
+
+
+class FilePreviewApi(Resource):
+    """
+    Service API File Preview endpoint
+
+    Provides secure file preview/download functionality for external API users.
+    Files can only be accessed if they belong to messages within the requesting app's context.
+    """
+
+    @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.QUERY))
+    def get(self, app_model: App, end_user: EndUser, file_id: str):
+        """
+        Preview/Download a file that was uploaded via Service API
+
+        Args:
+            app_model: The authenticated app model
+            end_user: The authenticated end user (optional)
+            file_id: UUID of the file to preview
+
+        Query Parameters:
+            user: Optional user identifier
+            as_attachment: Boolean, whether to download as attachment (default: false)
+
+        Returns:
+            Stream response with file content
+
+        Raises:
+            FileNotFoundError: File does not exist
+            FileAccessDeniedError: File access denied (not owned by app)
+        """
+        file_id = str(file_id)
+
+        # Parse query parameters
+        parser = reqparse.RequestParser()
+        parser.add_argument("as_attachment", type=bool, required=False, default=False, location="args")
+        args = parser.parse_args()
+
+        # Validate file ownership and get file objects
+        message_file, upload_file = self._validate_file_ownership(file_id, app_model.id)
+
+        # Get file content generator
+        try:
+            generator = storage.load(upload_file.key, stream=True)
+        except Exception as e:
+            raise FileNotFoundError(f"Failed to load file content: {str(e)}")
+
+        # Build response with appropriate headers
+        response = self._build_file_response(generator, upload_file, args["as_attachment"])
+
+        return response
+
+    def _validate_file_ownership(self, file_id: str, app_id: str) -> tuple[MessageFile, UploadFile]:
+        """
+        Validate that the file belongs to a message within the requesting app's context
+
+        Security validations performed:
+        1. File exists in MessageFile table (was used in a conversation)
+        2. Message belongs to the requesting app
+        3. UploadFile record exists and is accessible
+        4. File tenant matches app tenant (additional security layer)
+
+        Args:
+            file_id: UUID of the file to validate
+            app_id: UUID of the requesting app
+
+        Returns:
+            Tuple of (MessageFile, UploadFile) if validation passes
+
+        Raises:
+            FileNotFoundError: File or related records not found
+            FileAccessDeniedError: File does not belong to the app's context
+        """
+        try:
+            # Input validation
+            if not file_id or not app_id:
+                raise FileAccessDeniedError("Invalid file or app identifier")
+
+            # First, find the MessageFile that references this upload file
+            message_file = db.session.query(MessageFile).where(MessageFile.upload_file_id == file_id).first()
+
+            if not message_file:
+                raise FileNotFoundError("File not found in message context")
+
+            # Get the message and verify it belongs to the requesting app
+            message = (
+                db.session.query(Message).where(Message.id == message_file.message_id, Message.app_id == app_id).first()
+            )
+
+            if not message:
+                raise FileAccessDeniedError("File access denied: not owned by requesting app")
+
+            # Get the actual upload file record
+            upload_file = db.session.query(UploadFile).where(UploadFile.id == file_id).first()
+
+            if not upload_file:
+                raise FileNotFoundError("Upload file record not found")
+
+            # Additional security: verify tenant isolation
+            app = db.session.query(App).where(App.id == app_id).first()
+            if app and upload_file.tenant_id != app.tenant_id:
+                raise FileAccessDeniedError("File access denied: tenant mismatch")
+
+            return message_file, upload_file
+
+        except (FileNotFoundError, FileAccessDeniedError):
+            # Re-raise our custom exceptions
+            raise
+        except Exception as e:
+            # Log unexpected errors for debugging
+            logger.exception(
+                "Unexpected error during file ownership validation",
+                extra={"file_id": file_id, "app_id": app_id, "error": str(e)},
+            )
+            raise FileAccessDeniedError("File access validation failed")
+
+    def _build_file_response(self, generator, upload_file: UploadFile, as_attachment: bool = False) -> Response:
+        """
+        Build Flask Response object with appropriate headers for file streaming
+
+        Args:
+            generator: File content generator from storage
+            upload_file: UploadFile database record
+            as_attachment: Whether to set Content-Disposition as attachment
+
+        Returns:
+            Flask Response object with streaming file content
+        """
+        response = Response(
+            generator,
+            mimetype=upload_file.mime_type,
+            direct_passthrough=True,
+            headers={},
+        )
+
+        # Add Content-Length if known
+        if upload_file.size and upload_file.size > 0:
+            response.headers["Content-Length"] = str(upload_file.size)
+
+        # Add Accept-Ranges header for audio/video files to support seeking
+        if upload_file.mime_type in [
+            "audio/mpeg",
+            "audio/wav",
+            "audio/mp4",
+            "audio/ogg",
+            "audio/flac",
+            "audio/aac",
+            "video/mp4",
+            "video/webm",
+            "video/quicktime",
+            "audio/x-m4a",
+        ]:
+            response.headers["Accept-Ranges"] = "bytes"
+
+        # Set Content-Disposition for downloads
+        if as_attachment and upload_file.name:
+            encoded_filename = quote(upload_file.name)
+            response.headers["Content-Disposition"] = f"attachment; filename*=UTF-8''{encoded_filename}"
+            # Override content-type for downloads to force download
+            response.headers["Content-Type"] = "application/octet-stream"
+
+        # Add caching headers for performance
+        response.headers["Cache-Control"] = "public, max-age=3600"  # Cache for 1 hour
+
+        return response
+
+
+# Register the API endpoint
+api.add_resource(FilePreviewApi, "/files/<uuid:file_id>/preview")

api/controllers/service_api/app/workflow.py

@@ -5,7 +5,7 @@ from flask import request
 from flask_restful import Resource, fields, marshal_with, reqparse
 from flask_restful.inputs import int_range
 from sqlalchemy.orm import Session, sessionmaker
-from werkzeug.exceptions import InternalServerError
+from werkzeug.exceptions import BadRequest, InternalServerError, NotFound
 
 from controllers.service_api import api
 from controllers.service_api.app.error import (
@@ -34,6 +34,7 @@ from libs.helper import TimestampField
 from models.model import App, AppMode, EndUser
 from repositories.factory import DifyAPIRepositoryFactory
 from services.app_generate_service import AppGenerateService
+from services.errors.app import IsDraftWorkflowError, WorkflowIdFormatError, WorkflowNotFoundError
 from services.errors.llm import InvokeRateLimitError
 from services.workflow_app_service import WorkflowAppService
 
@@ -120,6 +121,59 @@ class WorkflowRunApi(Resource):
             raise InternalServerError()
 
 
+class WorkflowRunByIdApi(Resource):
+    @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True))
+    def post(self, app_model: App, end_user: EndUser, workflow_id: str):
+        """
+        Run specific workflow by ID
+        """
+        app_mode = AppMode.value_of(app_model.mode)
+        if app_mode != AppMode.WORKFLOW:
+            raise NotWorkflowAppError()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument("inputs", type=dict, required=True, nullable=False, location="json")
+        parser.add_argument("files", type=list, required=False, location="json")
+        parser.add_argument("response_mode", type=str, choices=["blocking", "streaming"], location="json")
+        args = parser.parse_args()
+
+        # Add workflow_id to args for AppGenerateService
+        args["workflow_id"] = workflow_id
+
+        external_trace_id = get_external_trace_id(request)
+        if external_trace_id:
+            args["external_trace_id"] = external_trace_id
+        streaming = args.get("response_mode") == "streaming"
+
+        try:
+            response = AppGenerateService.generate(
+                app_model=app_model, user=end_user, args=args, invoke_from=InvokeFrom.SERVICE_API, streaming=streaming
+            )
+
+            return helper.compact_generate_response(response)
+        except WorkflowNotFoundError as ex:
+            raise NotFound(str(ex))
+        except IsDraftWorkflowError as ex:
+            raise BadRequest(str(ex))
+        except WorkflowIdFormatError as ex:
+            raise BadRequest(str(ex))
+        except ProviderTokenNotInitError as ex:
+            raise ProviderNotInitializeError(ex.description)
+        except QuotaExceededError:
+            raise ProviderQuotaExceededError()
+        except ModelCurrentlyNotSupportError:
+            raise ProviderModelCurrentlyNotSupportError()
+        except InvokeRateLimitError as ex:
+            raise InvokeRateLimitHttpError(ex.description)
+        except InvokeError as e:
+            raise CompletionRequestError(e.description)
+        except ValueError as e:
+            raise e
+        except Exception:
+            logging.exception("internal server error.")
+            raise InternalServerError()
+
+
 class WorkflowTaskStopApi(Resource):
     @validate_app_token(fetch_user_arg=FetchUserArg(fetch_from=WhereisUserArg.JSON, required=True))
     def post(self, app_model: App, end_user: EndUser, task_id: str):
@@ -193,5 +247,6 @@ class WorkflowAppLogApi(Resource):
 
 api.add_resource(WorkflowRunApi, "/workflows/run")
 api.add_resource(WorkflowRunDetailApi, "/workflows/run/<string:workflow_run_id>")
+api.add_resource(WorkflowRunByIdApi, "/workflows/<string:workflow_id>/run")
 api.add_resource(WorkflowTaskStopApi, "/workflows/tasks/<string:task_id>/stop")
 api.add_resource(WorkflowAppLogApi, "/workflows/logs")

api/controllers/service_api/dataset/dataset.py

@@ -29,7 +29,7 @@ def _validate_name(name):
 
 
 def _validate_description_length(description):
-    if len(description) > 400:
+    if description and len(description) > 400:
         raise ValueError("Description cannot exceed 400 characters.")
     return description
 
@@ -87,7 +87,7 @@ class DatasetListApi(DatasetApiResource):
         )
         parser.add_argument(
             "description",
-            type=str,
+            type=_validate_description_length,
             nullable=True,
             required=False,
             default="",

api/controllers/service_api/dataset/document.py

@@ -234,8 +234,6 @@ class DocumentAddByFileApi(DatasetApiResource):
                 args["retrieval_model"].get("reranking_model").get("reranking_model_name"),
             )
 
-        # save file info
-        file = request.files["file"]
         # check file
         if "file" not in request.files:
             raise NoFileUploadedError()
@@ -243,6 +241,8 @@ class DocumentAddByFileApi(DatasetApiResource):
         if len(request.files) > 1:
             raise TooManyFilesError()
 
+        # save file info
+        file = request.files["file"]
         if not file.filename:
             raise FilenameNotExistsError
 
@@ -358,39 +358,6 @@ class DocumentUpdateByFileApi(DatasetApiResource):
         return documents_and_batch_fields, 200
 
 
-class DocumentDeleteApi(DatasetApiResource):
-    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
-    def delete(self, tenant_id, dataset_id, document_id):
-        """Delete document."""
-        document_id = str(document_id)
-        dataset_id = str(dataset_id)
-        tenant_id = str(tenant_id)
-
-        # get dataset info
-        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
-
-        if not dataset:
-            raise ValueError("Dataset does not exist.")
-
-        document = DocumentService.get_document(dataset.id, document_id)
-
-        # 404 if document not found
-        if document is None:
-            raise NotFound("Document Not Exists.")
-
-        # 403 if document is archived
-        if DocumentService.check_archived(document):
-            raise ArchivedDocumentImmutableError()
-
-        try:
-            # delete document
-            DocumentService.delete_document(document)
-        except services.errors.document.DocumentIndexingError:
-            raise DocumentIndexingError("Cannot delete document during indexing.")
-
-        return 204
-
-
 class DocumentListApi(DatasetApiResource):
     def get(self, tenant_id, dataset_id):
         dataset_id = str(dataset_id)
@@ -473,7 +440,7 @@ class DocumentIndexingStatusApi(DatasetApiResource):
         return data
 
 
-class DocumentDetailApi(DatasetApiResource):
+class DocumentApi(DatasetApiResource):
     METADATA_CHOICES = {"all", "only", "without"}
 
     def get(self, tenant_id, dataset_id, document_id):
@@ -567,6 +534,37 @@ class DocumentDetailApi(DatasetApiResource):
 
         return response
 
+    @cloud_edition_billing_rate_limit_check("knowledge", "dataset")
+    def delete(self, tenant_id, dataset_id, document_id):
+        """Delete document."""
+        document_id = str(document_id)
+        dataset_id = str(dataset_id)
+        tenant_id = str(tenant_id)
+
+        # get dataset info
+        dataset = db.session.query(Dataset).where(Dataset.tenant_id == tenant_id, Dataset.id == dataset_id).first()
+
+        if not dataset:
+            raise ValueError("Dataset does not exist.")
+
+        document = DocumentService.get_document(dataset.id, document_id)
+
+        # 404 if document not found
+        if document is None:
+            raise NotFound("Document Not Exists.")
+
+        # 403 if document is archived
+        if DocumentService.check_archived(document):
+            raise ArchivedDocumentImmutableError()
+
+        try:
+            # delete document
+            DocumentService.delete_document(document)
+        except services.errors.document.DocumentIndexingError:
+            raise DocumentIndexingError("Cannot delete document during indexing.")
+
+        return 204
+
 
 api.add_resource(
     DocumentAddByTextApi,
@@ -588,7 +586,6 @@ api.add_resource(
     "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update_by_file",
     "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>/update-by-file",
 )
-api.add_resource(DocumentDeleteApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
+api.add_resource(DocumentApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")
 api.add_resource(DocumentListApi, "/datasets/<uuid:dataset_id>/documents")
 api.add_resource(DocumentIndexingStatusApi, "/datasets/<uuid:dataset_id>/documents/<string:batch>/indexing-status")
-api.add_resource(DocumentDetailApi, "/datasets/<uuid:dataset_id>/documents/<uuid:document_id>")

api/controllers/web/app.py

@@ -1,5 +1,6 @@
 from flask import request
 from flask_restful import Resource, marshal_with, reqparse
+from werkzeug.exceptions import Unauthorized
 
 from controllers.common import fields
 from controllers.web import api
@@ -75,14 +76,14 @@ class AppWebAuthPermission(Resource):
         try:
             auth_header = request.headers.get("Authorization")
             if auth_header is None:
-                raise
+                raise Unauthorized("Authorization header is missing.")
             if " " not in auth_header:
-                raise
+                raise Unauthorized("Invalid Authorization header format. Expected 'Bearer <api-key>' format.")
 
             auth_scheme, tk = auth_header.split(None, 1)
             auth_scheme = auth_scheme.lower()
             if auth_scheme != "bearer":
-                raise
+                raise Unauthorized("Authorization scheme must be 'Bearer'")
 
             decoded = PassportService().verify(tk)
             user_id = decoded.get("user_id", "visitor")

api/controllers/web/files.py

@@ -12,18 +12,17 @@ from services.file_service import FileService
 class FileApi(WebApiResource):
     @marshal_with(file_fields)
     def post(self, app_model, end_user):
-        file = request.files["file"]
-        source = request.form.get("source")
-
         if "file" not in request.files:
             raise NoFileUploadedError()
 
         if len(request.files) > 1:
             raise TooManyFilesError()
 
+        file = request.files["file"]
         if not file.filename:
             raise FilenameNotExistsError
 
+        source = request.form.get("source")
         if source not in ("datasets", None):
             source = None
 

api/core/app/apps/advanced_chat/app_runner.py

@@ -118,26 +118,8 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
             ):
                 return
 
-            # Init conversation variables
-            stmt = select(ConversationVariable).where(
-                ConversationVariable.app_id == self.conversation.app_id,
-                ConversationVariable.conversation_id == self.conversation.id,
-            )
-            with Session(db.engine) as session:
-                db_conversation_variables = session.scalars(stmt).all()
-                if not db_conversation_variables:
-                    # Create conversation variables if they don't exist.
-                    db_conversation_variables = [
-                        ConversationVariable.from_variable(
-                            app_id=self.conversation.app_id, conversation_id=self.conversation.id, variable=variable
-                        )
-                        for variable in self._workflow.conversation_variables
-                    ]
-                    session.add_all(db_conversation_variables)
-                # Convert database entities to variables.
-                conversation_variables = [item.to_variable() for item in db_conversation_variables]
-
-                session.commit()
+            # Initialize conversation variables
+            conversation_variables = self._initialize_conversation_variables()
 
             # Create a variable pool.
             system_inputs = SystemVariable(
@@ -292,3 +274,100 @@ class AdvancedChatAppRunner(WorkflowBasedAppRunner):
             message_id=message_id,
             trace_manager=app_generate_entity.trace_manager,
         )
+
+    def _initialize_conversation_variables(self) -> list[VariableUnion]:
+        """
+        Initialize conversation variables for the current conversation.
+
+        This method:
+        1. Loads existing variables from the database
+        2. Creates new variables if none exist
+        3. Syncs missing variables from the workflow definition
+
+        :return: List of conversation variables ready for use
+        """
+        with Session(db.engine) as session:
+            existing_variables = self._load_existing_conversation_variables(session)
+
+            if not existing_variables:
+                # First time initialization - create all variables
+                existing_variables = self._create_all_conversation_variables(session)
+            else:
+                # Check and add any missing variables from the workflow
+                existing_variables = self._sync_missing_conversation_variables(session, existing_variables)
+
+            # Convert to Variable objects for use in the workflow
+            conversation_variables = [var.to_variable() for var in existing_variables]
+
+            session.commit()
+            return cast(list[VariableUnion], conversation_variables)
+
+    def _load_existing_conversation_variables(self, session: Session) -> list[ConversationVariable]:
+        """
+        Load existing conversation variables from the database.
+
+        :param session: Database session
+        :return: List of existing conversation variables
+        """
+        stmt = select(ConversationVariable).where(
+            ConversationVariable.app_id == self.conversation.app_id,
+            ConversationVariable.conversation_id == self.conversation.id,
+        )
+        return list(session.scalars(stmt).all())
+
+    def _create_all_conversation_variables(self, session: Session) -> list[ConversationVariable]:
+        """
+        Create all conversation variables for a new conversation.
+
+        :param session: Database session
+        :return: List of created conversation variables
+        """
+        new_variables = [
+            ConversationVariable.from_variable(
+                app_id=self.conversation.app_id, conversation_id=self.conversation.id, variable=variable
+            )
+            for variable in self._workflow.conversation_variables
+        ]
+
+        if new_variables:
+            session.add_all(new_variables)
+
+        return new_variables
+
+    def _sync_missing_conversation_variables(
+        self, session: Session, existing_variables: list[ConversationVariable]
+    ) -> list[ConversationVariable]:
+        """
+        Sync missing conversation variables from the workflow definition.
+
+        This handles the case where new variables are added to a workflow
+        after conversations have already been created.
+
+        :param session: Database session
+        :param existing_variables: List of existing conversation variables
+        :return: Updated list including any newly created variables
+        """
+        # Get IDs of existing and workflow variables
+        existing_ids = {var.id for var in existing_variables}
+        workflow_variables = {var.id: var for var in self._workflow.conversation_variables}
+
+        # Find missing variable IDs
+        missing_ids = set(workflow_variables.keys()) - existing_ids
+
+        if not missing_ids:
+            return existing_variables
+
+        # Create missing variables with their default values
+        new_variables = [
+            ConversationVariable.from_variable(
+                app_id=self.conversation.app_id,
+                conversation_id=self.conversation.id,
+                variable=workflow_variables[var_id],
+            )
+            for var_id in missing_ids
+        ]
+
+        session.add_all(new_variables)
+
+        # Return combined list
+        return existing_variables + new_variables

api/core/app/entities/app_invoke_entities.py

@@ -9,7 +9,6 @@ from core.app.app_config.entities import EasyUIBasedAppConfig, WorkflowUIBasedAp
 from core.entities.provider_configuration import ProviderModelBundle
 from core.file import File, FileUploadConfig
 from core.model_runtime.entities.model_entities import AIModelEntity
-from core.ops.ops_trace_manager import TraceQueueManager
 
 
 class InvokeFrom(Enum):
@@ -114,7 +113,8 @@ class AppGenerateEntity(BaseModel):
     extras: dict[str, Any] = Field(default_factory=dict)
 
     # tracing instance
-    trace_manager: Optional[TraceQueueManager] = None
+    # Using Any to avoid circular import with TraceQueueManager
+    trace_manager: Optional[Any] = None
 
 
 class EasyUIBasedAppGenerateEntity(AppGenerateEntity):

api/core/app/task_pipeline/message_cycle_manager.py

@@ -23,6 +23,7 @@ from core.app.entities.task_entities import (
     MessageFileStreamResponse,
     MessageReplaceStreamResponse,
     MessageStreamResponse,
+    StreamEvent,
     WorkflowTaskState,
 )
 from core.llm_generator.llm_generator import LLMGenerator
@@ -180,11 +181,15 @@ class MessageCycleManager:
         :param message_id: message id
         :return:
         """
+        message_file = db.session.query(MessageFile).filter(MessageFile.id == message_id).first()
+        event_type = StreamEvent.MESSAGE_FILE if message_file else StreamEvent.MESSAGE
+
         return MessageStreamResponse(
             task_id=self._application_generate_entity.task_id,
             id=message_id,
             answer=answer,
             from_variable_selector=from_variable_selector,
+            event=event_type,
         )
 
     def message_replace_to_stream_response(self, answer: str, reason: str = "") -> MessageReplaceStreamResponse:

api/core/entities/provider_configuration.py

@@ -843,7 +843,7 @@ class ProviderConfiguration(BaseModel):
                     continue
 
                 status = ModelStatus.ACTIVE
-                if m.model in model_setting_map:
+                if m.model_type in model_setting_map and m.model in model_setting_map[m.model_type]:
                     model_setting = model_setting_map[m.model_type][m.model]
                     if model_setting.enabled is False:
                         status = ModelStatus.DISABLED

api/core/entities/provider_entities.py

@@ -176,7 +176,7 @@ class ProviderConfig(BasicProviderConfig):
 
     scope: AppSelectorScope | ModelSelectorScope | ToolSelectorScope | None = None
     required: bool = False
-    default: Optional[Union[int, str]] = None
+    default: Optional[Union[int, str, float, bool]] = None
     options: Optional[list[Option]] = None
     label: Optional[I18nObject] = None
     help: Optional[I18nObject] = None

api/core/file/file_manager.py

@@ -32,7 +32,7 @@ def get_attr(*, file: File, attr: FileAttribute):
         case FileAttribute.TRANSFER_METHOD:
             return file.transfer_method.value
         case FileAttribute.URL:
-            return file.remote_url
+            return _to_url(file)
         case FileAttribute.EXTENSION:
             return file.extension
         case FileAttribute.RELATED_ID:

api/core/memory/token_buffer_memory.py

@@ -121,9 +121,8 @@ class TokenBufferMemory:
         curr_message_tokens = self.model_instance.get_llm_num_tokens(prompt_messages)
 
         if curr_message_tokens > max_token_limit:
-            pruned_memory = []
             while curr_message_tokens > max_token_limit and len(prompt_messages) > 1:
-                pruned_memory.append(prompt_messages.pop(0))
+                prompt_messages.pop(0)
                 curr_message_tokens = self.model_instance.get_llm_num_tokens(prompt_messages)
 
         return prompt_messages

api/core/plugin/impl/base.py

@@ -208,6 +208,7 @@ class BasePluginClient:
                     except Exception:
                         raise PluginDaemonInnerError(code=rep.code, message=rep.message)
 
+                    logger.error("Error in stream reponse for plugin %s", rep.__dict__)
                     self._handle_plugin_daemon_error(error.error_type, error.message)
                 raise ValueError(f"plugin daemon: {rep.message}, code: {rep.code}")
             if rep.data is None:

api/core/plugin/impl/exc.py

@@ -2,6 +2,8 @@ from collections.abc import Mapping
 
 from pydantic import TypeAdapter
 
+from extensions.ext_logging import get_request_id
+
 
 class PluginDaemonError(Exception):
     """Base class for all plugin daemon errors."""
@@ -11,7 +13,7 @@ class PluginDaemonError(Exception):
 
     def __str__(self) -> str:
         # returns the class name and description
-        return f"{self.__class__.__name__}: {self.description}"
+        return f"req_id: {get_request_id()} {self.__class__.__name__}: {self.description}"
 
 
 class PluginDaemonInternalError(PluginDaemonError):

api/core/rag/datasource/vdb/clickzetta/README.md

@@ -0,0 +1,190 @@
+# Clickzetta Vector Database Integration
+
+This module provides integration with Clickzetta Lakehouse as a vector database for Dify.
+
+## Features
+
+- **Vector Storage**: Store and retrieve high-dimensional vectors using Clickzetta's native VECTOR type
+- **Vector Search**: Efficient similarity search using HNSW algorithm
+- **Full-Text Search**: Leverage Clickzetta's inverted index for powerful text search capabilities
+- **Hybrid Search**: Combine vector similarity and full-text search for better results
+- **Multi-language Support**: Built-in support for Chinese, English, and Unicode text processing
+- **Scalable**: Leverage Clickzetta's distributed architecture for large-scale deployments
+
+## Configuration
+
+### Required Environment Variables
+
+All seven configuration parameters are required:
+
+```bash
+# Authentication
+CLICKZETTA_USERNAME=your_username
+CLICKZETTA_PASSWORD=your_password
+
+# Instance configuration
+CLICKZETTA_INSTANCE=your_instance_id
+CLICKZETTA_SERVICE=api.clickzetta.com
+CLICKZETTA_WORKSPACE=your_workspace
+CLICKZETTA_VCLUSTER=your_vcluster
+CLICKZETTA_SCHEMA=your_schema
+```
+
+### Optional Configuration
+
+```bash
+# Batch processing
+CLICKZETTA_BATCH_SIZE=100
+
+# Full-text search configuration
+CLICKZETTA_ENABLE_INVERTED_INDEX=true
+CLICKZETTA_ANALYZER_TYPE=chinese  # Options: keyword, english, chinese, unicode
+CLICKZETTA_ANALYZER_MODE=smart    # Options: max_word, smart
+
+# Vector search configuration
+CLICKZETTA_VECTOR_DISTANCE_FUNCTION=cosine_distance  # Options: l2_distance, cosine_distance
+```
+
+## Usage
+
+### 1. Set Clickzetta as the Vector Store
+
+In your Dify configuration, set:
+
+```bash
+VECTOR_STORE=clickzetta
+```
+
+### 2. Table Structure
+
+Clickzetta will automatically create tables with the following structure:
+
+```sql
+CREATE TABLE <collection_name> (
+    id STRING NOT NULL,
+    content STRING NOT NULL,
+    metadata JSON,
+    vector VECTOR(FLOAT, <dimension>) NOT NULL,
+    PRIMARY KEY (id)
+);
+
+-- Vector index for similarity search
+CREATE VECTOR INDEX idx_<collection_name>_vec
+ON TABLE <schema>.<collection_name>(vector) 
+PROPERTIES (
+    "distance.function" = "cosine_distance",
+    "scalar.type" = "f32"
+);
+
+-- Inverted index for full-text search (if enabled)
+CREATE INVERTED INDEX idx_<collection_name>_text
+ON <schema>.<collection_name>(content)
+PROPERTIES (
+    "analyzer" = "chinese",
+    "mode" = "smart"
+);
+```
+
+## Full-Text Search Capabilities
+
+Clickzetta supports advanced full-text search with multiple analyzers:
+
+### Analyzer Types
+
+1. **keyword**: No tokenization, treats the entire string as a single token
+   - Best for: Exact matching, IDs, codes
+
+2. **english**: Designed for English text
+   - Features: Recognizes ASCII letters and numbers, converts to lowercase
+   - Best for: English content
+
+3. **chinese**: Chinese text tokenizer
+   - Features: Recognizes Chinese and English characters, removes punctuation
+   - Best for: Chinese or mixed Chinese-English content
+
+4. **unicode**: Multi-language tokenizer based on Unicode
+   - Features: Recognizes text boundaries in multiple languages
+   - Best for: Multi-language content
+
+### Analyzer Modes
+
+- **max_word**: Fine-grained tokenization (more tokens)
+- **smart**: Intelligent tokenization (balanced)
+
+### Full-Text Search Functions
+
+- `MATCH_ALL(column, query)`: All terms must be present
+- `MATCH_ANY(column, query)`: At least one term must be present
+- `MATCH_PHRASE(column, query)`: Exact phrase matching
+- `MATCH_PHRASE_PREFIX(column, query)`: Phrase prefix matching
+- `MATCH_REGEXP(column, pattern)`: Regular expression matching
+
+## Performance Optimization
+
+### Vector Search
+
+1. **Adjust exploration factor** for accuracy vs speed trade-off:
+   ```sql
+   SET cz.vector.index.search.ef=64;
+   ```
+
+2. **Use appropriate distance functions**:
+   - `cosine_distance`: Best for normalized embeddings (e.g., from language models)
+   - `l2_distance`: Best for raw feature vectors
+
+### Full-Text Search
+
+1. **Choose the right analyzer**:
+   - Use `keyword` for exact matching
+   - Use language-specific analyzers for better tokenization
+
+2. **Combine with vector search**:
+   - Pre-filter with full-text search for better performance
+   - Use hybrid search for improved relevance
+
+## Troubleshooting
+
+### Connection Issues
+
+1. Verify all 7 required configuration parameters are set
+2. Check network connectivity to Clickzetta service
+3. Ensure the user has proper permissions on the schema
+
+### Search Performance
+
+1. Verify vector index exists:
+   ```sql
+   SHOW INDEX FROM <schema>.<table_name>;
+   ```
+
+2. Check if vector index is being used:
+   ```sql
+   EXPLAIN SELECT ... WHERE l2_distance(...) < threshold;
+   ```
+   Look for `vector_index_search_type` in the execution plan.
+
+### Full-Text Search Not Working
+
+1. Verify inverted index is created
+2. Check analyzer configuration matches your content language
+3. Use `TOKENIZE()` function to test tokenization:
+   ```sql
+   SELECT TOKENIZE('your text', map('analyzer', 'chinese', 'mode', 'smart'));
+   ```
+
+## Limitations
+
+1. Vector operations don't support `ORDER BY` or `GROUP BY` directly on vector columns
+2. Full-text search relevance scores are not provided by Clickzetta
+3. Inverted index creation may fail for very large existing tables (continue without error)
+4. Index naming constraints:
+   - Index names must be unique within a schema
+   - Only one vector index can be created per column
+   - The implementation uses timestamps to ensure unique index names
+5. A column can only have one vector index at a time
+
+## References
+
+- [Clickzetta Vector Search Documentation](https://yunqi.tech/documents/vector-search)
+- [Clickzetta Inverted Index Documentation](https://yunqi.tech/documents/inverted-index)
+- [Clickzetta SQL Functions](https://yunqi.tech/documents/sql-reference)

api/core/rag/datasource/vdb/clickzetta/__init__.py

@@ -0,0 +1 @@
+# Clickzetta Vector Database Integration for Dify

api/core/rag/datasource/vdb/elasticsearch/elasticsearch_vector.py

@@ -7,6 +7,7 @@ from urllib.parse import urlparse
 import requests
 from elasticsearch import Elasticsearch
 from flask import current_app
+from packaging.version import parse as parse_version
 from pydantic import BaseModel, model_validator
 
 from core.rag.datasource.vdb.field import Field
@@ -149,7 +150,7 @@ class ElasticSearchVector(BaseVector):
         return cast(str, info["version"]["number"])
 
     def _check_version(self):
-        if self._version < "8.0.0":
+        if parse_version(self._version) < parse_version("8.0.0"):
             raise ValueError("Elasticsearch vector database version must be greater than 8.0.0")
 
     def get_type(self) -> str:

api/core/rag/datasource/vdb/tencent/tencent_vector.py

@@ -246,6 +246,10 @@ class TencentVector(BaseVector):
         return self._get_search_res(res, score_threshold)
 
     def search_by_full_text(self, query: str, **kwargs: Any) -> list[Document]:
+        document_ids_filter = kwargs.get("document_ids_filter")
+        filter = None
+        if document_ids_filter:
+            filter = Filter(Filter.In("metadata.document_id", document_ids_filter))
         if not self._enable_hybrid_search:
             return []
         res = self._client.hybrid_search(
@@ -269,6 +273,7 @@ class TencentVector(BaseVector):
             ),
             retrieve_vector=False,
             limit=kwargs.get("top_k", 4),
+            filter=filter,
         )
         score_threshold = float(kwargs.get("score_threshold") or 0.0)
         return self._get_search_res(res, score_threshold)

api/core/rag/datasource/vdb/vector_factory.py

@@ -172,6 +172,10 @@ class Vector:
                 from core.rag.datasource.vdb.matrixone.matrixone_vector import MatrixoneVectorFactory
 
                 return MatrixoneVectorFactory
+            case VectorType.CLICKZETTA:
+                from core.rag.datasource.vdb.clickzetta.clickzetta_vector import ClickzettaVectorFactory
+
+                return ClickzettaVectorFactory
             case _:
                 raise ValueError(f"Vector store {vector_type} is not supported.")
 

api/core/rag/datasource/vdb/vector_type.py

@@ -30,3 +30,4 @@ class VectorType(StrEnum):
     TABLESTORE = "tablestore"
     HUAWEI_CLOUD = "huawei_cloud"
     MATRIXONE = "matrixone"
+    CLICKZETTA = "clickzetta"

api/core/rag/extractor/word_extractor.py

@@ -62,7 +62,7 @@ class WordExtractor(BaseExtractor):
 
     def extract(self) -> list[Document]:
         """Load given path as single page."""
-        content = self.parse_docx(self.file_path, "storage")
+        content = self.parse_docx(self.file_path)
         return [
             Document(
                 page_content=content,
@@ -189,23 +189,8 @@ class WordExtractor(BaseExtractor):
                 paragraph_content.append(run.text)
         return "".join(paragraph_content).strip()
 
-    def _parse_paragraph(self, paragraph, image_map):
-        paragraph_content = []
-        for run in paragraph.runs:
-            if run.element.xpath(".//a:blip"):
-                for blip in run.element.xpath(".//a:blip"):
-                    embed_id = blip.get("{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed")
-                    if embed_id:
-                        rel_target = run.part.rels[embed_id].target_ref
-                        if rel_target in image_map:
-                            paragraph_content.append(image_map[rel_target])
-            if run.text.strip():
-                paragraph_content.append(run.text.strip())
-        return " ".join(paragraph_content) if paragraph_content else ""
-
-    def parse_docx(self, docx_path, image_folder):
+    def parse_docx(self, docx_path):
         doc = DocxDocument(docx_path)
-        os.makedirs(image_folder, exist_ok=True)
 
         content = []
 

api/core/rag/splitter/fixed_text_splitter.py

@@ -5,14 +5,13 @@ from __future__ import annotations
 from typing import Any, Optional
 
 from core.model_manager import ModelInstance
-from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenzier import GPT2Tokenizer
+from core.model_runtime.model_providers.__base.tokenizers.gpt2_tokenizer import GPT2Tokenizer
 from core.rag.splitter.text_splitter import (
     TS,
     Collection,
     Literal,
     RecursiveCharacterTextSplitter,
     Set,
-    TokenTextSplitter,
     Union,
 )
 
@@ -45,14 +44,6 @@ class EnhanceRecursiveCharacterTextSplitter(RecursiveCharacterTextSplitter):
 
             return [len(text) for text in texts]
 
-        if issubclass(cls, TokenTextSplitter):
-            extra_kwargs = {
-                "model_name": embedding_model_instance.model if embedding_model_instance else "gpt2",
-                "allowed_special": allowed_special,
-                "disallowed_special": disallowed_special,
-            }
-            kwargs = {**kwargs, **extra_kwargs}
-
         return cls(length_function=_character_encoder, **kwargs)
 
 

api/core/tools/__base/tool.py

@@ -20,9 +20,6 @@ class Tool(ABC):
     The base class of a tool
     """
 
-    entity: ToolEntity
-    runtime: ToolRuntime
-
     def __init__(self, entity: ToolEntity, runtime: ToolRuntime) -> None:
         self.entity = entity
         self.runtime = runtime

api/core/tools/builtin_tool/providers/time/tools/localtime_to_timestamp.py

@@ -37,12 +37,12 @@ class LocaltimeToTimestampTool(BuiltinTool):
     @staticmethod
     def localtime_to_timestamp(localtime: str, time_format: str, local_tz=None) -> int | None:
         try:
-            if local_tz is None:
-                local_tz = datetime.now().astimezone().tzinfo
-            if isinstance(local_tz, str):
-                local_tz = pytz.timezone(local_tz)
             local_time = datetime.strptime(localtime, time_format)
-            localtime = local_tz.localize(local_time)  # type: ignore
+            if local_tz is None:
+                localtime = local_time.astimezone()  # type: ignore
+            elif isinstance(local_tz, str):
+                local_tz = pytz.timezone(local_tz)
+                localtime = local_tz.localize(local_time)  # type: ignore
             timestamp = int(localtime.timestamp())  # type: ignore
             return timestamp
         except Exception as e:

api/core/tools/builtin_tool/tool.py

@@ -20,8 +20,6 @@ class BuiltinTool(Tool):
     :param meta: the meta data of a tool call processing
     """
 
-    provider: str
-
     def __init__(self, provider: str, **kwargs):
         super().__init__(**kwargs)
         self.provider = provider

api/core/tools/custom_tool/tool.py

@@ -1,7 +1,8 @@
 import json
 from collections.abc import Generator
+from dataclasses import dataclass
 from os import getenv
-from typing import Any, Optional
+from typing import Any, Optional, Union
 from urllib.parse import urlencode
 
 import httpx
@@ -20,10 +21,21 @@ API_TOOL_DEFAULT_TIMEOUT = (
 )
 
 
-class ApiTool(Tool):
-    api_bundle: ApiToolBundle
-    provider_id: str
+@dataclass
+class ParsedResponse:
+    """Represents a parsed HTTP response with type information"""
 
+    content: Union[str, dict]
+    is_json: bool
+
+    def to_string(self) -> str:
+        """Convert response to string format for credential validation"""
+        if isinstance(self.content, dict):
+            return json.dumps(self.content, ensure_ascii=False)
+        return str(self.content)
+
+
+class ApiTool(Tool):
     """
     Api tool
     """
@@ -61,7 +73,9 @@ class ApiTool(Tool):
 
         response = self.do_http_request(self.api_bundle.server_url, self.api_bundle.method, headers, parameters)
         # validate response
-        return self.validate_and_parse_response(response)
+        parsed_response = self.validate_and_parse_response(response)
+        # For credential validation, always return as string
+        return parsed_response.to_string()
 
     def tool_provider_type(self) -> ToolProviderType:
         return ToolProviderType.API
@@ -115,23 +129,36 @@ class ApiTool(Tool):
 
         return headers
 
-    def validate_and_parse_response(self, response: httpx.Response) -> str:
+    def validate_and_parse_response(self, response: httpx.Response) -> ParsedResponse:
         """
-        validate the response
+        validate the response and return parsed content with type information
+
+        :return: ParsedResponse with content and is_json flag
         """
         if isinstance(response, httpx.Response):
             if response.status_code >= 400:
                 raise ToolInvokeError(f"Request failed with status code {response.status_code} and {response.text}")
             if not response.content:
-                return "Empty response from the tool, please check your parameters and try again."
+                return ParsedResponse(
+                    "Empty response from the tool, please check your parameters and try again.", False
+                )
+
+            # Check content type
+            content_type = response.headers.get("content-type", "").lower()
+            is_json_content_type = "application/json" in content_type
+
+            # Try to parse as JSON
             try:
-                response = response.json()
-                try:
-                    return json.dumps(response, ensure_ascii=False)
-                except Exception:
-                    return json.dumps(response)
+                json_response = response.json()
+                # If content-type indicates JSON, return as JSON object
+                if is_json_content_type:
+                    return ParsedResponse(json_response, True)
+                else:
+                    # If content-type doesn't indicate JSON, treat as text regardless of content
+                    return ParsedResponse(response.text, False)
             except Exception:
-                return response.text
+                # Not valid JSON, return as text
+                return ParsedResponse(response.text, False)
         else:
             raise ValueError(f"Invalid response type {type(response)}")
 
@@ -372,7 +399,14 @@ class ApiTool(Tool):
         response = self.do_http_request(self.api_bundle.server_url, self.api_bundle.method, headers, tool_parameters)
 
         # validate response
-        response = self.validate_and_parse_response(response)
+        parsed_response = self.validate_and_parse_response(response)
 
-        # assemble invoke message
-        yield self.create_text_message(response)
+        # assemble invoke message based on response type
+        if parsed_response.is_json and isinstance(parsed_response.content, dict):
+            yield self.create_json_message(parsed_response.content)
+        else:
+            # Convert to string if needed and create text message
+            text_response = (
+                parsed_response.content if isinstance(parsed_response.content, str) else str(parsed_response.content)
+            )
+            yield self.create_text_message(text_response)

api/core/tools/mcp_tool/tool.py

@@ -8,23 +8,16 @@ from core.mcp.mcp_client import MCPClient
 from core.mcp.types import ImageContent, TextContent
 from core.tools.__base.tool import Tool
 from core.tools.__base.tool_runtime import ToolRuntime
-from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, ToolParameter, ToolProviderType
+from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, ToolProviderType
 
 
 class MCPTool(Tool):
-    tenant_id: str
-    icon: str
-    runtime_parameters: Optional[list[ToolParameter]]
-    server_url: str
-    provider_id: str
-
     def __init__(
         self, entity: ToolEntity, runtime: ToolRuntime, tenant_id: str, icon: str, server_url: str, provider_id: str
     ) -> None:
         super().__init__(entity, runtime)
         self.tenant_id = tenant_id
         self.icon = icon
-        self.runtime_parameters = None
         self.server_url = server_url
         self.provider_id = provider_id
 

api/core/tools/plugin_tool/tool.py

@@ -9,11 +9,6 @@ from core.tools.entities.tool_entities import ToolEntity, ToolInvokeMessage, Too
 
 
 class PluginTool(Tool):
-    tenant_id: str
-    icon: str
-    plugin_unique_identifier: str
-    runtime_parameters: Optional[list[ToolParameter]]
-
     def __init__(
         self, entity: ToolEntity, runtime: ToolRuntime, tenant_id: str, icon: str, plugin_unique_identifier: str
     ) -> None:
@@ -21,7 +16,7 @@ class PluginTool(Tool):
         self.tenant_id = tenant_id
         self.icon = icon
         self.plugin_unique_identifier = plugin_unique_identifier
-        self.runtime_parameters = None
+        self.runtime_parameters: Optional[list[ToolParameter]] = None
 
     def tool_provider_type(self) -> ToolProviderType:
         return ToolProviderType.PLUGIN

api/core/tools/tool_engine.py

@@ -29,7 +29,7 @@ from core.tools.errors import (
     ToolProviderCredentialValidationError,
     ToolProviderNotFoundError,
 )
-from core.tools.utils.message_transformer import ToolFileMessageTransformer
+from core.tools.utils.message_transformer import ToolFileMessageTransformer, safe_json_value
 from core.tools.workflow_as_tool.tool import WorkflowTool
 from extensions.ext_database import db
 from models.enums import CreatorUserRole
@@ -247,7 +247,8 @@ class ToolEngine:
                 )
             elif response.type == ToolInvokeMessage.MessageType.JSON:
                 result += json.dumps(
-                    cast(ToolInvokeMessage.JsonMessage, response.message).json_object, ensure_ascii=False
+                    safe_json_value(cast(ToolInvokeMessage.JsonMessage, response.message).json_object),
+                    ensure_ascii=False,
                 )
             else:
                 result += str(response.message)

api/core/tools/tool_manager.py

@@ -7,6 +7,7 @@ from os import listdir, path
 from threading import Lock
 from typing import TYPE_CHECKING, Any, Literal, Optional, Union, cast
 
+import sqlalchemy as sa
 from pydantic import TypeAdapter
 from yarl import URL
 
@@ -616,7 +617,7 @@ class ToolManager:
                 WHERE tenant_id = :tenant_id
                 ORDER BY tenant_id, provider, is_default DESC, created_at DESC
                 """
-        ids = [row.id for row in db.session.execute(db.text(sql), {"tenant_id": tenant_id}).all()]
+        ids = [row.id for row in db.session.execute(sa.text(sql), {"tenant_id": tenant_id}).all()]
         return db.session.query(BuiltinToolProvider).where(BuiltinToolProvider.id.in_(ids)).all()
 
     @classmethod

api/core/tools/utils/dataset_retriever_tool.py

@@ -20,8 +20,6 @@ from core.tools.utils.dataset_retriever.dataset_retriever_base_tool import Datas
 
 
 class DatasetRetrieverTool(Tool):
-    retrieval_tool: DatasetRetrieverBaseTool
-
     def __init__(self, entity: ToolEntity, runtime: ToolRuntime, retrieval_tool: DatasetRetrieverBaseTool) -> None:
         super().__init__(entity, runtime)
         self.retrieval_tool = retrieval_tool

api/core/tools/utils/message_transformer.py

@@ -1,7 +1,14 @@
 import logging
 from collections.abc import Generator
+from datetime import date, datetime
+from decimal import Decimal
 from mimetypes import guess_extension
-from typing import Optional
+from typing import Optional, cast
+from uuid import UUID
+
+import numpy as np
+import pytz
+from flask_login import current_user
 
 from core.file import File, FileTransferMethod, FileType
 from core.tools.entities.tool_entities import ToolInvokeMessage
@@ -10,6 +17,41 @@ from core.tools.tool_file_manager import ToolFileManager
 logger = logging.getLogger(__name__)
 
 
+def safe_json_value(v):
+    if isinstance(v, datetime):
+        tz_name = getattr(current_user, "timezone", None) if current_user is not None else None
+        if not tz_name:
+            tz_name = "UTC"
+        return v.astimezone(pytz.timezone(tz_name)).isoformat()
+    elif isinstance(v, date):
+        return v.isoformat()
+    elif isinstance(v, UUID):
+        return str(v)
+    elif isinstance(v, Decimal):
+        return float(v)
+    elif isinstance(v, bytes):
+        try:
+            return v.decode("utf-8")
+        except UnicodeDecodeError:
+            return v.hex()
+    elif isinstance(v, memoryview):
+        return v.tobytes().hex()
+    elif isinstance(v, np.ndarray):
+        return v.tolist()
+    elif isinstance(v, dict):
+        return safe_json_dict(v)
+    elif isinstance(v, list | tuple | set):
+        return [safe_json_value(i) for i in v]
+    else:
+        return v
+
+
+def safe_json_dict(d):
+    if not isinstance(d, dict):
+        raise TypeError("safe_json_dict() expects a dictionary (dict) as input")
+    return {k: safe_json_value(v) for k, v in d.items()}
+
+
 class ToolFileMessageTransformer:
     @classmethod
     def transform_tool_invoke_messages(
@@ -113,6 +155,12 @@ class ToolFileMessageTransformer:
                             )
                     else:
                         yield message
+
+            elif message.type == ToolInvokeMessage.MessageType.JSON:
+                if isinstance(message.message, ToolInvokeMessage.JsonMessage):
+                    json_msg = cast(ToolInvokeMessage.JsonMessage, message.message)
+                    json_msg.json_object = safe_json_value(json_msg.json_object)
+                yield message
             else:
                 yield message
 

api/core/tools/workflow_as_tool/tool.py

@@ -25,15 +25,6 @@ logger = logging.getLogger(__name__)
 
 
 class WorkflowTool(Tool):
-    workflow_app_id: str
-    version: str
-    workflow_entities: dict[str, Any]
-    workflow_call_depth: int
-    thread_pool_id: Optional[str] = None
-    workflow_as_tool_id: str
-
-    label: str
-
     """
     Workflow tool.
     """

api/core/variables/segments.py

@@ -119,6 +119,13 @@ class ObjectSegment(Segment):
 
 
 class ArraySegment(Segment):
+    @property
+    def text(self) -> str:
+        # Return empty string for empty arrays instead of "[]"
+        if not self.value:
+            return ""
+        return super().text
+
     @property
     def markdown(self) -> str:
         items = []
@@ -155,6 +162,9 @@ class ArrayStringSegment(ArraySegment):
 
     @property
     def text(self) -> str:
+        # Return empty string for empty arrays instead of "[]"
+        if not self.value:
+            return ""
         return json.dumps(self.value, ensure_ascii=False)
 
 

api/core/variables/types.py

@@ -109,7 +109,7 @@ class SegmentType(StrEnum):
         elif array_validation == ArrayValidation.FIRST:
             return element_type.is_valid(value[0])
         else:
-            return all([element_type.is_valid(i, array_validation=ArrayValidation.NONE)] for i in value)
+            return all(element_type.is_valid(i, array_validation=ArrayValidation.NONE) for i in value)
 
     def is_valid(self, value: Any, array_validation: ArrayValidation = ArrayValidation.FIRST) -> bool:
         """
@@ -152,7 +152,7 @@ class SegmentType(StrEnum):
 
 
 _ARRAY_ELEMENT_TYPES_MAPPING: Mapping[SegmentType, SegmentType] = {
-    # ARRAY_ANY does not have correpond element type.
+    # ARRAY_ANY does not have corresponding element type.
     SegmentType.ARRAY_STRING: SegmentType.STRING,
     SegmentType.ARRAY_NUMBER: SegmentType.NUMBER,
     SegmentType.ARRAY_OBJECT: SegmentType.OBJECT,

api/core/workflow/nodes/document_extractor/node.py

@@ -168,7 +168,57 @@ def _extract_text_by_mime_type(*, file_content: bytes, mime_type: str) -> str:
 def _extract_text_by_file_extension(*, file_content: bytes, file_extension: str) -> str:
     """Extract text from a file based on its file extension."""
     match file_extension:
-        case ".txt" | ".markdown" | ".md" | ".html" | ".htm" | ".xml":
+        case (
+            ".txt"
+            | ".markdown"
+            | ".md"
+            | ".html"
+            | ".htm"
+            | ".xml"
+            | ".c"
+            | ".h"
+            | ".cpp"
+            | ".hpp"
+            | ".cc"
+            | ".cxx"
+            | ".c++"
+            | ".py"
+            | ".js"
+            | ".ts"
+            | ".jsx"
+            | ".tsx"
+            | ".java"
+            | ".php"
+            | ".rb"
+            | ".go"
+            | ".rs"
+            | ".swift"
+            | ".kt"
+            | ".scala"
+            | ".sh"
+            | ".bash"
+            | ".bat"
+            | ".ps1"
+            | ".sql"
+            | ".r"
+            | ".m"
+            | ".pl"
+            | ".lua"
+            | ".vim"
+            | ".asm"
+            | ".s"
+            | ".css"
+            | ".scss"
+            | ".less"
+            | ".sass"
+            | ".ini"
+            | ".cfg"
+            | ".conf"
+            | ".toml"
+            | ".env"
+            | ".log"
+            | ".vtt"
+        ):
             return _extract_text_from_plain_text(file_content)
         case ".json":
             return _extract_text_from_json(file_content)
@@ -194,8 +244,6 @@ def _extract_text_by_file_extension(*, file_content: bytes, file_extension: str)
             return _extract_text_from_eml(file_content)
         case ".msg":
             return _extract_text_from_msg(file_content)
-        case ".vtt":
-            return _extract_text_from_vtt(file_content)
         case ".properties":
             return _extract_text_from_properties(file_content)
         case _:

api/core/workflow/nodes/http_request/executor.py

@@ -91,7 +91,7 @@ class Executor:
         self.auth = node_data.authorization
         self.timeout = timeout
         self.ssl_verify = node_data.ssl_verify
-        self.params = []
+        self.params = None
         self.headers = {}
         self.content = None
         self.files = None
@@ -139,7 +139,8 @@ class Executor:
                 (self.variable_pool.convert_template(key).text, self.variable_pool.convert_template(value_str).text)
             )
 
-        self.params = result
+        if result:
+            self.params = result
 
     def _init_headers(self):
         """

api/core/workflow/nodes/tool/tool_node.py

@@ -318,6 +318,33 @@ class ToolNode(BaseNode):
                     json.append(message.message.json_object)
             elif message.type == ToolInvokeMessage.MessageType.LINK:
                 assert isinstance(message.message, ToolInvokeMessage.TextMessage)
+
+                if message.meta:
+                    transfer_method = message.meta.get("transfer_method", FileTransferMethod.TOOL_FILE)
+                else:
+                    transfer_method = FileTransferMethod.TOOL_FILE
+
+                tool_file_id = message.message.text.split("/")[-1].split(".")[0]
+
+                with Session(db.engine) as session:
+                    stmt = select(ToolFile).where(ToolFile.id == tool_file_id)
+                    tool_file = session.scalar(stmt)
+                    if tool_file is None:
+                        raise ToolFileError(f"Tool file {tool_file_id} does not exist")
+
+                mapping = {
+                    "tool_file_id": tool_file_id,
+                    "type": file_factory.get_file_type_by_mime_type(tool_file.mimetype),
+                    "transfer_method": transfer_method,
+                    "url": message.message.text,
+                }
+
+                file = file_factory.build_from_mapping(
+                    mapping=mapping,
+                    tenant_id=self.tenant_id,
+                )
+                files.append(file)
+
                 stream_text = f"Link: {message.message.text}\n"
                 text += stream_text
                 yield RunStreamChunkEvent(chunk_content=stream_text, from_variable_selector=[node_id, "text"])

api/extensions/ext_otel.py

@@ -136,6 +136,8 @@ def init_app(app: DifyApp):
     from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter as HTTPSpanExporter
     from opentelemetry.instrumentation.celery import CeleryInstrumentor
     from opentelemetry.instrumentation.flask import FlaskInstrumentor
+    from opentelemetry.instrumentation.redis import RedisInstrumentor
+    from opentelemetry.instrumentation.requests import RequestsInstrumentor
     from opentelemetry.instrumentation.sqlalchemy import SQLAlchemyInstrumentor
     from opentelemetry.metrics import get_meter, get_meter_provider, set_meter_provider
     from opentelemetry.propagate import set_global_textmap
@@ -234,6 +236,8 @@ def init_app(app: DifyApp):
         CeleryInstrumentor(tracer_provider=get_tracer_provider(), meter_provider=get_meter_provider()).instrument()
     instrument_exception_logging()
     init_sqlalchemy_instrumentor(app)
+    RedisInstrumentor().instrument()
+    RequestsInstrumentor().instrument()
     atexit.register(shutdown_tracer)
 
 

api/extensions/ext_storage.py

@@ -69,6 +69,19 @@ class Storage:
                 from extensions.storage.supabase_storage import SupabaseStorage
 
                 return SupabaseStorage
+            case StorageType.CLICKZETTA_VOLUME:
+                from extensions.storage.clickzetta_volume.clickzetta_volume_storage import (
+                    ClickZettaVolumeConfig,
+                    ClickZettaVolumeStorage,
+                )
+
+                def create_clickzetta_volume_storage():
+                    # ClickZettaVolumeConfig will automatically read from environment variables
+                    # and fallback to CLICKZETTA_* config if CLICKZETTA_VOLUME_* is not set
+                    volume_config = ClickZettaVolumeConfig()
+                    return ClickZettaVolumeStorage(volume_config)
+
+                return create_clickzetta_volume_storage
             case _:
                 raise ValueError(f"unsupported storage type {storage_type}")
 

api/extensions/storage/clickzetta_volume/__init__.py

@@ -0,0 +1,5 @@
+"""ClickZetta Volume storage implementation."""
+
+from .clickzetta_volume_storage import ClickZettaVolumeStorage
+
+__all__ = ["ClickZettaVolumeStorage"]

api/extensions/storage/clickzetta_volume/clickzetta_volume_storage.py

@@ -0,0 +1,530 @@
+"""ClickZetta Volume Storage Implementation
+
+This module provides storage backend using ClickZetta Volume functionality.
+Supports Table Volume, User Volume, and External Volume types.
+"""
+
+import logging
+import os
+import tempfile
+from collections.abc import Generator
+from io import BytesIO
+from pathlib import Path
+from typing import Optional
+
+import clickzetta  # type: ignore[import]
+from pydantic import BaseModel, model_validator
+
+from extensions.storage.base_storage import BaseStorage
+
+from .volume_permissions import VolumePermissionManager, check_volume_permission
+
+logger = logging.getLogger(__name__)
+
+
+class ClickZettaVolumeConfig(BaseModel):
+    """Configuration for ClickZetta Volume storage."""
+
+    username: str = ""
+    password: str = ""
+    instance: str = ""
+    service: str = "api.clickzetta.com"
+    workspace: str = "quick_start"
+    vcluster: str = "default_ap"
+    schema_name: str = "dify"
+    volume_type: str = "table"  # table|user|external
+    volume_name: Optional[str] = None  # For external volumes
+    table_prefix: str = "dataset_"  # Prefix for table volume names
+    dify_prefix: str = "dify_km"  # Directory prefix for User Volume
+    permission_check: bool = True  # Enable/disable permission checking
+
+    @model_validator(mode="before")
+    @classmethod
+    def validate_config(cls, values: dict) -> dict:
+        """Validate the configuration values.
+
+        This method will first try to use CLICKZETTA_VOLUME_* environment variables,
+        then fall back to CLICKZETTA_* environment variables (for vector DB config).
+        """
+        import os
+
+        # Helper function to get environment variable with fallback
+        def get_env_with_fallback(volume_key: str, fallback_key: str, default: str | None = None) -> str:
+            # First try CLICKZETTA_VOLUME_* specific config
+            volume_value = values.get(volume_key.lower().replace("clickzetta_volume_", ""))
+            if volume_value:
+                return str(volume_value)
+
+            # Then try environment variables
+            volume_env = os.getenv(volume_key)
+            if volume_env:
+                return volume_env
+
+            # Fall back to existing CLICKZETTA_* config
+            fallback_env = os.getenv(fallback_key)
+            if fallback_env:
+                return fallback_env
+
+            return default or ""
+
+        # Apply environment variables with fallback to existing CLICKZETTA_* config
+        values.setdefault("username", get_env_with_fallback("CLICKZETTA_VOLUME_USERNAME", "CLICKZETTA_USERNAME"))
+        values.setdefault("password", get_env_with_fallback("CLICKZETTA_VOLUME_PASSWORD", "CLICKZETTA_PASSWORD"))
+        values.setdefault("instance", get_env_with_fallback("CLICKZETTA_VOLUME_INSTANCE", "CLICKZETTA_INSTANCE"))
+        values.setdefault(
+            "service", get_env_with_fallback("CLICKZETTA_VOLUME_SERVICE", "CLICKZETTA_SERVICE", "api.clickzetta.com")
+        )
+        values.setdefault(
+            "workspace", get_env_with_fallback("CLICKZETTA_VOLUME_WORKSPACE", "CLICKZETTA_WORKSPACE", "quick_start")
+        )
+        values.setdefault(
+            "vcluster", get_env_with_fallback("CLICKZETTA_VOLUME_VCLUSTER", "CLICKZETTA_VCLUSTER", "default_ap")
+        )
+        values.setdefault("schema_name", get_env_with_fallback("CLICKZETTA_VOLUME_SCHEMA", "CLICKZETTA_SCHEMA", "dify"))
+
+        # Volume-specific configurations (no fallback to vector DB config)
+        values.setdefault("volume_type", os.getenv("CLICKZETTA_VOLUME_TYPE", "table"))
+        values.setdefault("volume_name", os.getenv("CLICKZETTA_VOLUME_NAME"))
+        values.setdefault("table_prefix", os.getenv("CLICKZETTA_VOLUME_TABLE_PREFIX", "dataset_"))
+        values.setdefault("dify_prefix", os.getenv("CLICKZETTA_VOLUME_DIFY_PREFIX", "dify_km"))
+        # 暂时禁用权限检查功能，直接设置为false
+        values.setdefault("permission_check", False)
+
+        # Validate required fields
+        if not values.get("username"):
+            raise ValueError("CLICKZETTA_VOLUME_USERNAME or CLICKZETTA_USERNAME is required")
+        if not values.get("password"):
+            raise ValueError("CLICKZETTA_VOLUME_PASSWORD or CLICKZETTA_PASSWORD is required")
+        if not values.get("instance"):
+            raise ValueError("CLICKZETTA_VOLUME_INSTANCE or CLICKZETTA_INSTANCE is required")
+
+        # Validate volume type
+        volume_type = values["volume_type"]
+        if volume_type not in ["table", "user", "external"]:
+            raise ValueError("CLICKZETTA_VOLUME_TYPE must be one of: table, user, external")
+
+        if volume_type == "external" and not values.get("volume_name"):
+            raise ValueError("CLICKZETTA_VOLUME_NAME is required for external volume type")
+
+        return values
+
+
+class ClickZettaVolumeStorage(BaseStorage):
+    """ClickZetta Volume storage implementation."""
+
+    def __init__(self, config: ClickZettaVolumeConfig):
+        """Initialize ClickZetta Volume storage.
+
+        Args:
+            config: ClickZetta Volume configuration
+        """
+        self._config = config
+        self._connection = None
+        self._permission_manager: VolumePermissionManager | None = None
+        self._init_connection()
+        self._init_permission_manager()
+
+        logger.info("ClickZetta Volume storage initialized with type: %s", config.volume_type)
+
+    def _init_connection(self):
+        """Initialize ClickZetta connection."""
+        try:
+            self._connection = clickzetta.connect(
+                username=self._config.username,
+                password=self._config.password,
+                instance=self._config.instance,
+                service=self._config.service,
+                workspace=self._config.workspace,
+                vcluster=self._config.vcluster,
+                schema=self._config.schema_name,
+            )
+            logger.debug("ClickZetta connection established")
+        except Exception as e:
+            logger.exception("Failed to connect to ClickZetta")
+            raise
+
+    def _init_permission_manager(self):
+        """Initialize permission manager."""
+        try:
+            self._permission_manager = VolumePermissionManager(
+                self._connection, self._config.volume_type, self._config.volume_name
+            )
+            logger.debug("Permission manager initialized")
+        except Exception as e:
+            logger.exception("Failed to initialize permission manager")
+            raise
+
+    def _get_volume_path(self, filename: str, dataset_id: Optional[str] = None) -> str:
+        """Get the appropriate volume path based on volume type."""
+        if self._config.volume_type == "user":
+            # Add dify prefix for User Volume to organize files
+            return f"{self._config.dify_prefix}/{filename}"
+        elif self._config.volume_type == "table":
+            # Check if this should use User Volume (special directories)
+            if dataset_id in ["upload_files", "temp", "cache", "tools", "website_files", "privkeys"]:
+                # Use User Volume with dify prefix for special directories
+                return f"{self._config.dify_prefix}/{filename}"
+
+            if dataset_id:
+                return f"{self._config.table_prefix}{dataset_id}/{filename}"
+            else:
+                # Extract dataset_id from filename if not provided
+                # Format: dataset_id/filename
+                if "/" in filename:
+                    return filename
+                else:
+                    raise ValueError("dataset_id is required for table volume or filename must include dataset_id/")
+        elif self._config.volume_type == "external":
+            return filename
+        else:
+            raise ValueError(f"Unsupported volume type: {self._config.volume_type}")
+
+    def _get_volume_sql_prefix(self, dataset_id: Optional[str] = None) -> str:
+        """Get SQL prefix for volume operations."""
+        if self._config.volume_type == "user":
+            return "USER VOLUME"
+        elif self._config.volume_type == "table":
+            # For Dify's current file storage pattern, most files are stored in
+            # paths like "upload_files/tenant_id/uuid.ext", "tools/tenant_id/uuid.ext"
+            # These should use USER VOLUME for better compatibility
+            if dataset_id in ["upload_files", "temp", "cache", "tools", "website_files", "privkeys"]:
+                return "USER VOLUME"
+
+            # Only use TABLE VOLUME for actual dataset-specific paths
+            # like "dataset_12345/file.pdf" or paths with dataset_ prefix
+            if dataset_id:
+                table_name = f"{self._config.table_prefix}{dataset_id}"
+            else:
+                # Default table name for generic operations
+                table_name = "default_dataset"
+            return f"TABLE VOLUME {table_name}"
+        elif self._config.volume_type == "external":
+            return f"VOLUME {self._config.volume_name}"
+        else:
+            raise ValueError(f"Unsupported volume type: {self._config.volume_type}")
+
+    def _execute_sql(self, sql: str, fetch: bool = False):
+        """Execute SQL command."""
+        try:
+            if self._connection is None:
+                raise RuntimeError("Connection not initialized")
+            with self._connection.cursor() as cursor:
+                cursor.execute(sql)
+                if fetch:
+                    return cursor.fetchall()
+                return None
+        except Exception as e:
+            logger.exception("SQL execution failed: %s", sql)
+            raise
+
+    def _ensure_table_volume_exists(self, dataset_id: str) -> None:
+        """Ensure table volume exists for the given dataset_id."""
+        if self._config.volume_type != "table" or not dataset_id:
+            return
+
+        # Skip for upload_files and other special directories that use USER VOLUME
+        if dataset_id in ["upload_files", "temp", "cache", "tools", "website_files", "privkeys"]:
+            return
+
+        table_name = f"{self._config.table_prefix}{dataset_id}"
+
+        try:
+            # Check if table exists
+            check_sql = f"SHOW TABLES LIKE '{table_name}'"
+            result = self._execute_sql(check_sql, fetch=True)
+
+            if not result:
+                # Create table with volume
+                create_sql = f"""
+                CREATE TABLE {table_name} (
+                    id INT PRIMARY KEY AUTO_INCREMENT,
+                    filename VARCHAR(255) NOT NULL,
+                    created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
+                    updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
+                    INDEX idx_filename (filename)
+                ) WITH VOLUME
+                """
+                self._execute_sql(create_sql)
+                logger.info("Created table volume: %s", table_name)
+
+        except Exception as e:
+            logger.warning("Failed to create table volume %s: %s", table_name, e)
+            # Don't raise exception, let the operation continue
+            # The table might exist but not be visible due to permissions
+
+    def save(self, filename: str, data: bytes) -> None:
+        """Save data to ClickZetta Volume.
+
+        Args:
+            filename: File path in volume
+            data: File content as bytes
+        """
+        # Extract dataset_id from filename if present
+        dataset_id = None
+        if "/" in filename and self._config.volume_type == "table":
+            parts = filename.split("/", 1)
+            if parts[0].startswith(self._config.table_prefix):
+                dataset_id = parts[0][len(self._config.table_prefix) :]
+                filename = parts[1]
+            else:
+                dataset_id = parts[0]
+                filename = parts[1]
+
+        # Ensure table volume exists (for table volumes)
+        if dataset_id:
+            self._ensure_table_volume_exists(dataset_id)
+
+        # Check permissions (if enabled)
+        if self._config.permission_check:
+            # Skip permission check for special directories that use USER VOLUME
+            if dataset_id not in ["upload_files", "temp", "cache", "tools", "website_files", "privkeys"]:
+                if self._permission_manager is not None:
+                    check_volume_permission(self._permission_manager, "save", dataset_id)
+
+        # Write data to temporary file
+        with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+            temp_file.write(data)
+            temp_file_path = temp_file.name
+
+        try:
+            # Upload to volume
+            volume_prefix = self._get_volume_sql_prefix(dataset_id)
+
+            # Get the actual volume path (may include dify_km prefix)
+            volume_path = self._get_volume_path(filename, dataset_id)
+            actual_filename = volume_path.split("/")[-1] if "/" in volume_path else volume_path
+
+            # For User Volume, use the full path with dify_km prefix
+            if volume_prefix == "USER VOLUME":
+                sql = f"PUT '{temp_file_path}' TO {volume_prefix} FILE '{volume_path}'"
+            else:
+                sql = f"PUT '{temp_file_path}' TO {volume_prefix} FILE '{filename}'"
+
+            self._execute_sql(sql)
+            logger.debug("File %s saved to ClickZetta Volume at path %s", filename, volume_path)
+        finally:
+            # Clean up temporary file
+            Path(temp_file_path).unlink(missing_ok=True)
+
+    def load_once(self, filename: str) -> bytes:
+        """Load file content from ClickZetta Volume.
+
+        Args:
+            filename: File path in volume
+
+        Returns:
+            File content as bytes
+        """
+        # Extract dataset_id from filename if present
+        dataset_id = None
+        if "/" in filename and self._config.volume_type == "table":
+            parts = filename.split("/", 1)
+            if parts[0].startswith(self._config.table_prefix):
+                dataset_id = parts[0][len(self._config.table_prefix) :]
+                filename = parts[1]
+            else:
+                dataset_id = parts[0]
+                filename = parts[1]
+
+        # Check permissions (if enabled)
+        if self._config.permission_check:
+            # Skip permission check for special directories that use USER VOLUME
+            if dataset_id not in ["upload_files", "temp", "cache", "tools", "website_files", "privkeys"]:
+                if self._permission_manager is not None:
+                    check_volume_permission(self._permission_manager, "load_once", dataset_id)
+
+        # Download to temporary directory
+        with tempfile.TemporaryDirectory() as temp_dir:
+            volume_prefix = self._get_volume_sql_prefix(dataset_id)
+
+            # Get the actual volume path (may include dify_km prefix)
+            volume_path = self._get_volume_path(filename, dataset_id)
+
+            # For User Volume, use the full path with dify_km prefix
+            if volume_prefix == "USER VOLUME":
+                sql = f"GET {volume_prefix} FILE '{volume_path}' TO '{temp_dir}'"
+            else:
+                sql = f"GET {volume_prefix} FILE '{filename}' TO '{temp_dir}'"
+
+            self._execute_sql(sql)
+
+            # Find the downloaded file (may be in subdirectories)
+            downloaded_file = None
+            for root, dirs, files in os.walk(temp_dir):
+                for file in files:
+                    if file == filename or file == os.path.basename(filename):
+                        downloaded_file = Path(root) / file
+                        break
+                if downloaded_file:
+                    break
+
+            if not downloaded_file or not downloaded_file.exists():
+                raise FileNotFoundError(f"Downloaded file not found: {filename}")
+
+            content = downloaded_file.read_bytes()
+
+            logger.debug("File %s loaded from ClickZetta Volume", filename)
+            return content
+
+    def load_stream(self, filename: str) -> Generator:
+        """Load file as stream from ClickZetta Volume.
+
+        Args:
+            filename: File path in volume
+
+        Yields:
+            File content chunks
+        """
+        content = self.load_once(filename)
+        batch_size = 4096
+        stream = BytesIO(content)
+
+        while chunk := stream.read(batch_size):
+            yield chunk
+
+        logger.debug("File %s loaded as stream from ClickZetta Volume", filename)
+
+    def download(self, filename: str, target_filepath: str):
+        """Download file from ClickZetta Volume to local path.
+
+        Args:
+            filename: File path in volume
+            target_filepath: Local target file path
+        """
+        content = self.load_once(filename)
+
+        with Path(target_filepath).open("wb") as f:
+            f.write(content)
+
+        logger.debug("File %s downloaded from ClickZetta Volume to %s", filename, target_filepath)
+
+    def exists(self, filename: str) -> bool:
+        """Check if file exists in ClickZetta Volume.
+
+        Args:
+            filename: File path in volume
+
+        Returns:
+            True if file exists, False otherwise
+        """
+        try:
+            # Extract dataset_id from filename if present
+            dataset_id = None
+            if "/" in filename and self._config.volume_type == "table":
+                parts = filename.split("/", 1)
+                if parts[0].startswith(self._config.table_prefix):
+                    dataset_id = parts[0][len(self._config.table_prefix) :]
+                    filename = parts[1]
+                else:
+                    dataset_id = parts[0]
+                    filename = parts[1]
+
+            volume_prefix = self._get_volume_sql_prefix(dataset_id)
+
+            # Get the actual volume path (may include dify_km prefix)
+            volume_path = self._get_volume_path(filename, dataset_id)
+
+            # For User Volume, use the full path with dify_km prefix
+            if volume_prefix == "USER VOLUME":
+                sql = f"LIST {volume_prefix} REGEXP = '^{volume_path}$'"
+            else:
+                sql = f"LIST {volume_prefix} REGEXP = '^{filename}$'"
+
+            rows = self._execute_sql(sql, fetch=True)
+
+            exists = len(rows) > 0
+            logger.debug("File %s exists check: %s", filename, exists)
+            return exists
+        except Exception as e:
+            logger.warning("Error checking file existence for %s: %s", filename, e)
+            return False
+
+    def delete(self, filename: str):
+        """Delete file from ClickZetta Volume.
+
+        Args:
+            filename: File path in volume
+        """
+        if not self.exists(filename):
+            logger.debug("File %s not found, skip delete", filename)
+            return
+
+        # Extract dataset_id from filename if present
+        dataset_id = None
+        if "/" in filename and self._config.volume_type == "table":
+            parts = filename.split("/", 1)
+            if parts[0].startswith(self._config.table_prefix):
+                dataset_id = parts[0][len(self._config.table_prefix) :]
+                filename = parts[1]
+            else:
+                dataset_id = parts[0]
+                filename = parts[1]
+
+        volume_prefix = self._get_volume_sql_prefix(dataset_id)
+
+        # Get the actual volume path (may include dify_km prefix)
+        volume_path = self._get_volume_path(filename, dataset_id)
+
+        # For User Volume, use the full path with dify_km prefix
+        if volume_prefix == "USER VOLUME":
+            sql = f"REMOVE {volume_prefix} FILE '{volume_path}'"
+        else:
+            sql = f"REMOVE {volume_prefix} FILE '{filename}'"
+
+        self._execute_sql(sql)
+
+        logger.debug("File %s deleted from ClickZetta Volume", filename)
+
+    def scan(self, path: str, files: bool = True, directories: bool = False) -> list[str]:
+        """Scan files and directories in ClickZetta Volume.
+
+        Args:
+            path: Path to scan (dataset_id for table volumes)
+            files: Include files in results
+            directories: Include directories in results
+
+        Returns:
+            List of file/directory paths
+        """
+        try:
+            # For table volumes, path is treated as dataset_id
+            dataset_id = None
+            if self._config.volume_type == "table":
+                dataset_id = path
+                path = ""  # Root of the table volume
+
+            volume_prefix = self._get_volume_sql_prefix(dataset_id)
+
+            # For User Volume, add dify prefix to path
+            if volume_prefix == "USER VOLUME":
+                if path:
+                    scan_path = f"{self._config.dify_prefix}/{path}"
+                    sql = f"LIST {volume_prefix} SUBDIRECTORY '{scan_path}'"
+                else:
+                    sql = f"LIST {volume_prefix} SUBDIRECTORY '{self._config.dify_prefix}'"
+            else:
+                if path:
+                    sql = f"LIST {volume_prefix} SUBDIRECTORY '{path}'"
+                else:
+                    sql = f"LIST {volume_prefix}"
+
+            rows = self._execute_sql(sql, fetch=True)
+
+            result = []
+            for row in rows:
+                file_path = row[0]  # relative_path column
+
+                # For User Volume, remove dify prefix from results
+                dify_prefix_with_slash = f"{self._config.dify_prefix}/"
+                if volume_prefix == "USER VOLUME" and file_path.startswith(dify_prefix_with_slash):
+                    file_path = file_path[len(dify_prefix_with_slash) :]  # Remove prefix
+
+                if files and not file_path.endswith("/") or directories and file_path.endswith("/"):
+                    result.append(file_path)
+
+            logger.debug("Scanned %d items in path %s", len(result), path)
+            return result
+
+        except Exception as e:
+            logger.exception("Error scanning path %s", path)
+            return []

api/extensions/storage/clickzetta_volume/file_lifecycle.py

@@ -0,0 +1,516 @@
+"""ClickZetta Volume文件生命周期管理
+
+该模块提供文件版本控制、自动清理、备份和恢复等生命周期管理功能。
+支持知识库文件的完整生命周期管理。
+"""
+
+import json
+import logging
+from dataclasses import asdict, dataclass
+from datetime import datetime, timedelta
+from enum import Enum
+from typing import Any, Optional
+
+logger = logging.getLogger(__name__)
+
+
+class FileStatus(Enum):
+    """文件状态枚举"""
+
+    ACTIVE = "active"  # 活跃状态
+    ARCHIVED = "archived"  # 已归档
+    DELETED = "deleted"  # 已删除（软删除）
+    BACKUP = "backup"  # 备份文件
+
+
+@dataclass
+class FileMetadata:
+    """文件元数据"""
+
+    filename: str
+    size: int | None
+    created_at: datetime
+    modified_at: datetime
+    version: int | None
+    status: FileStatus
+    checksum: Optional[str] = None
+    tags: Optional[dict[str, str]] = None
+    parent_version: Optional[int] = None
+
+    def to_dict(self) -> dict:
+        """转换为字典格式"""
+        data = asdict(self)
+        data["created_at"] = self.created_at.isoformat()
+        data["modified_at"] = self.modified_at.isoformat()
+        data["status"] = self.status.value
+        return data
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "FileMetadata":
+        """从字典创建实例"""
+        data = data.copy()
+        data["created_at"] = datetime.fromisoformat(data["created_at"])
+        data["modified_at"] = datetime.fromisoformat(data["modified_at"])
+        data["status"] = FileStatus(data["status"])
+        return cls(**data)
+
+
+class FileLifecycleManager:
+    """文件生命周期管理器"""
+
+    def __init__(self, storage, dataset_id: Optional[str] = None):
+        """初始化生命周期管理器
+
+        Args:
+            storage: ClickZetta Volume存储实例
+            dataset_id: 数据集ID（用于Table Volume）
+        """
+        self._storage = storage
+        self._dataset_id = dataset_id
+        self._metadata_file = ".dify_file_metadata.json"
+        self._version_prefix = ".versions/"
+        self._backup_prefix = ".backups/"
+        self._deleted_prefix = ".deleted/"
+
+        # 获取权限管理器（如果存在）
+        self._permission_manager: Optional[Any] = getattr(storage, "_permission_manager", None)
+
+    def save_with_lifecycle(self, filename: str, data: bytes, tags: Optional[dict[str, str]] = None) -> FileMetadata:
+        """保存文件并管理生命周期
+
+        Args:
+            filename: 文件名
+            data: 文件内容
+            tags: 文件标签
+
+        Returns:
+            文件元数据
+        """
+        # 权限检查
+        if not self._check_permission(filename, "save"):
+            from .volume_permissions import VolumePermissionError
+
+            raise VolumePermissionError(
+                f"Permission denied for lifecycle save operation on file: {filename}",
+                operation="save",
+                volume_type=getattr(self._storage, "_config", {}).get("volume_type", "unknown"),
+                dataset_id=self._dataset_id,
+            )
+
+        try:
+            # 1. 检查是否存在旧版本
+            metadata_dict = self._load_metadata()
+            current_metadata = metadata_dict.get(filename)
+
+            # 2. 如果存在旧版本，创建版本备份
+            if current_metadata:
+                self._create_version_backup(filename, current_metadata)
+
+            # 3. 计算文件信息
+            now = datetime.now()
+            checksum = self._calculate_checksum(data)
+            new_version = (current_metadata["version"] + 1) if current_metadata else 1
+
+            # 4. 保存新文件
+            self._storage.save(filename, data)
+
+            # 5. 创建元数据
+            created_at = now
+            parent_version = None
+
+            if current_metadata:
+                # 如果created_at是字符串，转换为datetime
+                if isinstance(current_metadata["created_at"], str):
+                    created_at = datetime.fromisoformat(current_metadata["created_at"])
+                else:
+                    created_at = current_metadata["created_at"]
+                parent_version = current_metadata["version"]
+
+            file_metadata = FileMetadata(
+                filename=filename,
+                size=len(data),
+                created_at=created_at,
+                modified_at=now,
+                version=new_version,
+                status=FileStatus.ACTIVE,
+                checksum=checksum,
+                tags=tags or {},
+                parent_version=parent_version,
+            )
+
+            # 6. 更新元数据
+            metadata_dict[filename] = file_metadata.to_dict()
+            self._save_metadata(metadata_dict)
+
+            logger.info("File %s saved with lifecycle management, version %s", filename, new_version)
+            return file_metadata
+
+        except Exception as e:
+            logger.exception("Failed to save file with lifecycle")
+            raise
+
+    def get_file_metadata(self, filename: str) -> Optional[FileMetadata]:
+        """获取文件元数据
+
+        Args:
+            filename: 文件名
+
+        Returns:
+            文件元数据，如果不存在返回None
+        """
+        try:
+            metadata_dict = self._load_metadata()
+            if filename in metadata_dict:
+                return FileMetadata.from_dict(metadata_dict[filename])
+            return None
+        except Exception as e:
+            logger.exception("Failed to get file metadata for %s", filename)
+            return None
+
+    def list_file_versions(self, filename: str) -> list[FileMetadata]:
+        """列出文件的所有版本
+
+        Args:
+            filename: 文件名
+
+        Returns:
+            文件版本列表，按版本号排序
+        """
+        try:
+            versions = []
+
+            # 获取当前版本
+            current_metadata = self.get_file_metadata(filename)
+            if current_metadata:
+                versions.append(current_metadata)
+
+            # 获取历史版本
+            version_pattern = f"{self._version_prefix}{filename}.v*"
+            try:
+                version_files = self._storage.scan(self._dataset_id or "", files=True)
+                for file_path in version_files:
+                    if file_path.startswith(f"{self._version_prefix}{filename}.v"):
+                        # 解析版本号
+                        version_str = file_path.split(".v")[-1].split(".")[0]
+                        try:
+                            version_num = int(version_str)
+                            # 这里简化处理，实际应该从版本文件中读取元数据
+                            # 暂时创建基本的元数据信息
+                        except ValueError:
+                            continue
+            except:
+                # 如果无法扫描版本文件，只返回当前版本
+                pass
+
+            return sorted(versions, key=lambda x: x.version or 0, reverse=True)
+
+        except Exception as e:
+            logger.exception("Failed to list file versions for %s", filename)
+            return []
+
+    def restore_version(self, filename: str, version: int) -> bool:
+        """恢复文件到指定版本
+
+        Args:
+            filename: 文件名
+            version: 要恢复的版本号
+
+        Returns:
+            恢复是否成功
+        """
+        try:
+            version_filename = f"{self._version_prefix}{filename}.v{version}"
+
+            # 检查版本文件是否存在
+            if not self._storage.exists(version_filename):
+                logger.warning("Version %s of %s not found", version, filename)
+                return False
+
+            # 读取版本文件内容
+            version_data = self._storage.load_once(version_filename)
+
+            # 保存当前版本为备份
+            current_metadata = self.get_file_metadata(filename)
+            if current_metadata:
+                self._create_version_backup(filename, current_metadata.to_dict())
+
+            # 恢复文件
+            self.save_with_lifecycle(filename, version_data, {"restored_from": str(version)})
+            return True
+
+        except Exception as e:
+            logger.exception("Failed to restore %s to version %s", filename, version)
+            return False
+
+    def archive_file(self, filename: str) -> bool:
+        """归档文件
+
+        Args:
+            filename: 文件名
+
+        Returns:
+            归档是否成功
+        """
+        # 权限检查
+        if not self._check_permission(filename, "archive"):
+            logger.warning("Permission denied for archive operation on file: %s", filename)
+            return False
+
+        try:
+            # 更新文件状态为归档
+            metadata_dict = self._load_metadata()
+            if filename not in metadata_dict:
+                logger.warning("File %s not found in metadata", filename)
+                return False
+
+            metadata_dict[filename]["status"] = FileStatus.ARCHIVED.value
+            metadata_dict[filename]["modified_at"] = datetime.now().isoformat()
+
+            self._save_metadata(metadata_dict)
+
+            logger.info("File %s archived successfully", filename)
+            return True
+
+        except Exception as e:
+            logger.exception("Failed to archive file %s", filename)
+            return False
+
+    def soft_delete_file(self, filename: str) -> bool:
+        """软删除文件（移动到删除目录）
+
+        Args:
+            filename: 文件名
+
+        Returns:
+            删除是否成功
+        """
+        # 权限检查
+        if not self._check_permission(filename, "delete"):
+            logger.warning("Permission denied for soft delete operation on file: %s", filename)
+            return False
+
+        try:
+            # 检查文件是否存在
+            if not self._storage.exists(filename):
+                logger.warning("File %s not found", filename)
+                return False
+
+            # 读取文件内容
+            file_data = self._storage.load_once(filename)
+
+            # 移动到删除目录
+            deleted_filename = f"{self._deleted_prefix}{filename}.{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+            self._storage.save(deleted_filename, file_data)
+
+            # 删除原文件
+            self._storage.delete(filename)
+
+            # 更新元数据
+            metadata_dict = self._load_metadata()
+            if filename in metadata_dict:
+                metadata_dict[filename]["status"] = FileStatus.DELETED.value
+                metadata_dict[filename]["modified_at"] = datetime.now().isoformat()
+                self._save_metadata(metadata_dict)
+
+            logger.info("File %s soft deleted successfully", filename)
+            return True
+
+        except Exception as e:
+            logger.exception("Failed to soft delete file %s", filename)
+            return False
+
+    def cleanup_old_versions(self, max_versions: int = 5, max_age_days: int = 30) -> int:
+        """清理旧版本文件
+
+        Args:
+            max_versions: 保留的最大版本数
+            max_age_days: 版本文件的最大保留天数
+
+        Returns:
+            清理的文件数量
+        """
+        try:
+            cleaned_count = 0
+            cutoff_date = datetime.now() - timedelta(days=max_age_days)
+
+            # 获取所有版本文件
+            try:
+                all_files = self._storage.scan(self._dataset_id or "", files=True)
+                version_files = [f for f in all_files if f.startswith(self._version_prefix)]
+
+                # 按文件分组
+                file_versions: dict[str, list[tuple[int, str]]] = {}
+                for version_file in version_files:
+                    # 解析文件名和版本
+                    parts = version_file[len(self._version_prefix) :].split(".v")
+                    if len(parts) >= 2:
+                        base_filename = parts[0]
+                        version_part = parts[1].split(".")[0]
+                        try:
+                            version_num = int(version_part)
+                            if base_filename not in file_versions:
+                                file_versions[base_filename] = []
+                            file_versions[base_filename].append((version_num, version_file))
+                        except ValueError:
+                            continue
+
+                # 清理每个文件的旧版本
+                for base_filename, versions in file_versions.items():
+                    # 按版本号排序
+                    versions.sort(key=lambda x: x[0], reverse=True)
+
+                    # 保留最新的max_versions个版本，删除其余的
+                    if len(versions) > max_versions:
+                        to_delete = versions[max_versions:]
+                        for version_num, version_file in to_delete:
+                            self._storage.delete(version_file)
+                            cleaned_count += 1
+                            logger.debug("Cleaned old version: %s", version_file)
+
+                logger.info("Cleaned %d old version files", cleaned_count)
+
+            except Exception as e:
+                logger.warning("Could not scan for version files: %s", e)
+
+            return cleaned_count
+
+        except Exception as e:
+            logger.exception("Failed to cleanup old versions")
+            return 0
+
+    def get_storage_statistics(self) -> dict[str, Any]:
+        """获取存储统计信息
+
+        Returns:
+            存储统计字典
+        """
+        try:
+            metadata_dict = self._load_metadata()
+
+            stats: dict[str, Any] = {
+                "total_files": len(metadata_dict),
+                "active_files": 0,
+                "archived_files": 0,
+                "deleted_files": 0,
+                "total_size": 0,
+                "versions_count": 0,
+                "oldest_file": None,
+                "newest_file": None,
+            }
+
+            oldest_date = None
+            newest_date = None
+
+            for filename, metadata in metadata_dict.items():
+                file_meta = FileMetadata.from_dict(metadata)
+
+                # 统计文件状态
+                if file_meta.status == FileStatus.ACTIVE:
+                    stats["active_files"] = (stats["active_files"] or 0) + 1
+                elif file_meta.status == FileStatus.ARCHIVED:
+                    stats["archived_files"] = (stats["archived_files"] or 0) + 1
+                elif file_meta.status == FileStatus.DELETED:
+                    stats["deleted_files"] = (stats["deleted_files"] or 0) + 1
+
+                # 统计大小
+                stats["total_size"] = (stats["total_size"] or 0) + (file_meta.size or 0)
+
+                # 统计版本
+                stats["versions_count"] = (stats["versions_count"] or 0) + (file_meta.version or 0)
+
+                # 找出最新和最旧的文件
+                if oldest_date is None or file_meta.created_at < oldest_date:
+                    oldest_date = file_meta.created_at
+                    stats["oldest_file"] = filename
+
+                if newest_date is None or file_meta.modified_at > newest_date:
+                    newest_date = file_meta.modified_at
+                    stats["newest_file"] = filename
+
+            return stats
+
+        except Exception as e:
+            logger.exception("Failed to get storage statistics")
+            return {}
+
+    def _create_version_backup(self, filename: str, metadata: dict):
+        """创建版本备份"""
+        try:
+            # 读取当前文件内容
+            current_data = self._storage.load_once(filename)
+
+            # 保存为版本文件
+            version_filename = f"{self._version_prefix}{filename}.v{metadata['version']}"
+            self._storage.save(version_filename, current_data)
+
+            logger.debug("Created version backup: %s", version_filename)
+
+        except Exception as e:
+            logger.warning("Failed to create version backup for %s: %s", filename, e)
+
+    def _load_metadata(self) -> dict[str, Any]:
+        """加载元数据文件"""
+        try:
+            if self._storage.exists(self._metadata_file):
+                metadata_content = self._storage.load_once(self._metadata_file)
+                result = json.loads(metadata_content.decode("utf-8"))
+                return dict(result) if result else {}
+            else:
+                return {}
+        except Exception as e:
+            logger.warning("Failed to load metadata: %s", e)
+            return {}
+
+    def _save_metadata(self, metadata_dict: dict):
+        """保存元数据文件"""
+        try:
+            metadata_content = json.dumps(metadata_dict, indent=2, ensure_ascii=False)
+            self._storage.save(self._metadata_file, metadata_content.encode("utf-8"))
+            logger.debug("Metadata saved successfully")
+        except Exception as e:
+            logger.exception("Failed to save metadata")
+            raise
+
+    def _calculate_checksum(self, data: bytes) -> str:
+        """计算文件校验和"""
+        import hashlib
+
+        return hashlib.md5(data).hexdigest()
+
+    def _check_permission(self, filename: str, operation: str) -> bool:
+        """检查文件操作权限
+
+        Args:
+            filename: 文件名
+            operation: 操作类型
+
+        Returns:
+            True if permission granted, False otherwise
+        """
+        # 如果没有权限管理器，默认允许
+        if not self._permission_manager:
+            return True
+
+        try:
+            # 根据操作类型映射到权限
+            operation_mapping = {
+                "save": "save",
+                "load": "load_once",
+                "delete": "delete",
+                "archive": "delete",  # 归档需要删除权限
+                "restore": "save",  # 恢复需要写权限
+                "cleanup": "delete",  # 清理需要删除权限
+                "read": "load_once",
+                "write": "save",
+            }
+
+            mapped_operation = operation_mapping.get(operation, operation)
+
+            # 检查权限
+            result = self._permission_manager.validate_operation(mapped_operation, self._dataset_id)
+            return bool(result)
+
+        except Exception as e:
+            logger.exception("Permission check failed for %s operation %s", filename, operation)
+            # 安全默认：权限检查失败时拒绝访问
+            return False

api/extensions/storage/clickzetta_volume/volume_permissions.py

@@ -0,0 +1,646 @@
+"""ClickZetta Volume权限管理机制
+
+该模块提供Volume权限检查、验证和管理功能。
+根据ClickZetta的权限模型，不同Volume类型有不同的权限要求。
+"""
+
+import logging
+from enum import Enum
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+class VolumePermission(Enum):
+    """Volume权限类型枚举"""
+
+    READ = "SELECT"  # 对应ClickZetta的SELECT权限
+    WRITE = "INSERT,UPDATE,DELETE"  # 对应ClickZetta的写权限
+    LIST = "SELECT"  # 列出文件需要SELECT权限
+    DELETE = "INSERT,UPDATE,DELETE"  # 删除文件需要写权限
+    USAGE = "USAGE"  # External Volume需要的基本权限
+
+
+class VolumePermissionManager:
+    """Volume权限管理器"""
+
+    def __init__(self, connection_or_config, volume_type: str | None = None, volume_name: Optional[str] = None):
+        """初始化权限管理器
+
+        Args:
+            connection_or_config: ClickZetta连接对象或配置字典
+            volume_type: Volume类型 (user|table|external)
+            volume_name: Volume名称 (用于external volume)
+        """
+        # 支持两种初始化方式：连接对象或配置字典
+        if isinstance(connection_or_config, dict):
+            # 从配置字典创建连接
+            import clickzetta  # type: ignore[import-untyped]
+
+            config = connection_or_config
+            self._connection = clickzetta.connect(
+                username=config.get("username"),
+                password=config.get("password"),
+                instance=config.get("instance"),
+                service=config.get("service"),
+                workspace=config.get("workspace"),
+                vcluster=config.get("vcluster"),
+                schema=config.get("schema") or config.get("database"),
+            )
+            self._volume_type = config.get("volume_type", volume_type)
+            self._volume_name = config.get("volume_name", volume_name)
+        else:
+            # 直接使用连接对象
+            self._connection = connection_or_config
+            self._volume_type = volume_type
+            self._volume_name = volume_name
+
+        if not self._connection:
+            raise ValueError("Valid connection or config is required")
+        if not self._volume_type:
+            raise ValueError("volume_type is required")
+
+        self._permission_cache: dict[str, set[str]] = {}
+        self._current_username = None  # 将从连接中获取当前用户名
+
+    def check_permission(self, operation: VolumePermission, dataset_id: Optional[str] = None) -> bool:
+        """检查用户是否有执行特定操作的权限
+
+        Args:
+            operation: 要执行的操作类型
+            dataset_id: 数据集ID (用于table volume)
+
+        Returns:
+            True if user has permission, False otherwise
+        """
+        try:
+            if self._volume_type == "user":
+                return self._check_user_volume_permission(operation)
+            elif self._volume_type == "table":
+                return self._check_table_volume_permission(operation, dataset_id)
+            elif self._volume_type == "external":
+                return self._check_external_volume_permission(operation)
+            else:
+                logger.warning("Unknown volume type: %s", self._volume_type)
+                return False
+
+        except Exception as e:
+            logger.exception("Permission check failed")
+            return False
+
+    def _check_user_volume_permission(self, operation: VolumePermission) -> bool:
+        """检查User Volume权限
+
+        User Volume权限规则:
+        - 用户对自己的User Volume有全部权限
+        - 只要用户能够连接到ClickZetta，就默认具有User Volume的基本权限
+        - 更注重连接身份验证，而不是复杂的权限检查
+        """
+        try:
+            # 获取当前用户名
+            current_user = self._get_current_username()
+
+            # 检查基本连接状态
+            with self._connection.cursor() as cursor:
+                # 简单的连接测试，如果能执行查询说明用户有基本权限
+                cursor.execute("SELECT 1")
+                result = cursor.fetchone()
+
+                if result:
+                    logger.debug(
+                        "User Volume permission check for %s, operation %s: granted (basic connection verified)",
+                        current_user,
+                        operation.name,
+                    )
+                    return True
+                else:
+                    logger.warning(
+                        "User Volume permission check failed: cannot verify basic connection for %s", current_user
+                    )
+                    return False
+
+        except Exception as e:
+            logger.exception("User Volume permission check failed")
+            # 对于User Volume，如果权限检查失败，可能是配置问题，给出更友好的错误提示
+            logger.info("User Volume permission check failed, but permission checking is disabled in this version")
+            return False
+
+    def _check_table_volume_permission(self, operation: VolumePermission, dataset_id: Optional[str]) -> bool:
+        """检查Table Volume权限
+
+        Table Volume权限规则:
+        - Table Volume权限继承对应表的权限
+        - SELECT权限 -> 可以READ/LIST文件
+        - INSERT,UPDATE,DELETE权限 -> 可以WRITE/DELETE文件
+        """
+        if not dataset_id:
+            logger.warning("dataset_id is required for table volume permission check")
+            return False
+
+        table_name = f"dataset_{dataset_id}" if not dataset_id.startswith("dataset_") else dataset_id
+
+        try:
+            # 检查表权限
+            permissions = self._get_table_permissions(table_name)
+            required_permissions = set(operation.value.split(","))
+
+            # 检查是否有所需的所有权限
+            has_permission = required_permissions.issubset(permissions)
+
+            logger.debug(
+                "Table Volume permission check for %s, operation %s: required=%s, has=%s, granted=%s",
+                table_name,
+                operation.name,
+                required_permissions,
+                permissions,
+                has_permission,
+            )
+
+            return has_permission
+
+        except Exception as e:
+            logger.exception("Table volume permission check failed for %s", table_name)
+            return False
+
+    def _check_external_volume_permission(self, operation: VolumePermission) -> bool:
+        """检查External Volume权限
+
+        External Volume权限规则:
+        - 尝试获取对External Volume的权限
+        - 如果权限检查失败，进行备选验证
+        - 对于开发环境，提供更宽松的权限检查
+        """
+        if not self._volume_name:
+            logger.warning("volume_name is required for external volume permission check")
+            return False
+
+        try:
+            # 检查External Volume权限
+            permissions = self._get_external_volume_permissions(self._volume_name)
+
+            # External Volume权限映射：根据操作类型确定所需权限
+            required_permissions = set()
+
+            if operation in [VolumePermission.READ, VolumePermission.LIST]:
+                required_permissions.add("read")
+            elif operation in [VolumePermission.WRITE, VolumePermission.DELETE]:
+                required_permissions.add("write")
+
+            # 检查是否有所需的所有权限
+            has_permission = required_permissions.issubset(permissions)
+
+            logger.debug(
+                "External Volume permission check for %s, operation %s: required=%s, has=%s, granted=%s",
+                self._volume_name,
+                operation.name,
+                required_permissions,
+                permissions,
+                has_permission,
+            )
+
+            # 如果权限检查失败，尝试备选验证
+            if not has_permission:
+                logger.info("Direct permission check failed for %s, trying fallback verification", self._volume_name)
+
+                # 备选验证：尝试列出Volume来验证基本访问权限
+                try:
+                    with self._connection.cursor() as cursor:
+                        cursor.execute("SHOW VOLUMES")
+                        volumes = cursor.fetchall()
+                        for volume in volumes:
+                            if len(volume) > 0 and volume[0] == self._volume_name:
+                                logger.info("Fallback verification successful for %s", self._volume_name)
+                                return True
+                except Exception as fallback_e:
+                    logger.warning("Fallback verification failed for %s: %s", self._volume_name, fallback_e)
+
+            return has_permission
+
+        except Exception as e:
+            logger.exception("External volume permission check failed for %s", self._volume_name)
+            logger.info("External Volume permission check failed, but permission checking is disabled in this version")
+            return False
+
+    def _get_table_permissions(self, table_name: str) -> set[str]:
+        """获取用户对指定表的权限
+
+        Args:
+            table_name: 表名
+
+        Returns:
+            用户对该表的权限集合
+        """
+        cache_key = f"table:{table_name}"
+
+        if cache_key in self._permission_cache:
+            return self._permission_cache[cache_key]
+
+        permissions = set()
+
+        try:
+            with self._connection.cursor() as cursor:
+                # 使用正确的ClickZetta语法检查当前用户权限
+                cursor.execute("SHOW GRANTS")
+                grants = cursor.fetchall()
+
+                # 解析权限结果，查找对该表的权限
+                for grant in grants:
+                    if len(grant) >= 3:  # 典型格式: (privilege, object_type, object_name, ...)
+                        privilege = grant[0].upper()
+                        object_type = grant[1].upper() if len(grant) > 1 else ""
+                        object_name = grant[2] if len(grant) > 2 else ""
+
+                        # 检查是否是对该表的权限
+                        if (
+                            object_type == "TABLE"
+                            and object_name == table_name
+                            or object_type == "SCHEMA"
+                            and object_name in table_name
+                        ):
+                            if privilege in ["SELECT", "INSERT", "UPDATE", "DELETE", "ALL"]:
+                                if privilege == "ALL":
+                                    permissions.update(["SELECT", "INSERT", "UPDATE", "DELETE"])
+                                else:
+                                    permissions.add(privilege)
+
+                # 如果没有找到明确的权限，尝试执行一个简单的查询来验证权限
+                if not permissions:
+                    try:
+                        cursor.execute(f"SELECT COUNT(*) FROM {table_name} LIMIT 1")
+                        permissions.add("SELECT")
+                    except Exception:
+                        logger.debug("Cannot query table %s, no SELECT permission", table_name)
+
+        except Exception as e:
+            logger.warning("Could not check table permissions for %s: %s", table_name, e)
+            # 安全默认：权限检查失败时拒绝访问
+            pass
+
+        # 缓存权限信息
+        self._permission_cache[cache_key] = permissions
+        return permissions
+
+    def _get_current_username(self) -> str:
+        """获取当前用户名"""
+        if self._current_username:
+            return self._current_username
+
+        try:
+            with self._connection.cursor() as cursor:
+                cursor.execute("SELECT CURRENT_USER()")
+                result = cursor.fetchone()
+                if result:
+                    self._current_username = result[0]
+                    return str(self._current_username)
+        except Exception as e:
+            logger.exception("Failed to get current username")
+
+        return "unknown"
+
+    def _get_user_permissions(self, username: str) -> set[str]:
+        """获取用户的基本权限集合"""
+        cache_key = f"user_permissions:{username}"
+
+        if cache_key in self._permission_cache:
+            return self._permission_cache[cache_key]
+
+        permissions = set()
+
+        try:
+            with self._connection.cursor() as cursor:
+                # 使用正确的ClickZetta语法检查当前用户权限
+                cursor.execute("SHOW GRANTS")
+                grants = cursor.fetchall()
+
+                # 解析权限结果，查找用户的基本权限
+                for grant in grants:
+                    if len(grant) >= 3:  # 典型格式: (privilege, object_type, object_name, ...)
+                        privilege = grant[0].upper()
+                        object_type = grant[1].upper() if len(grant) > 1 else ""
+
+                        # 收集所有相关权限
+                        if privilege in ["SELECT", "INSERT", "UPDATE", "DELETE", "ALL"]:
+                            if privilege == "ALL":
+                                permissions.update(["SELECT", "INSERT", "UPDATE", "DELETE"])
+                            else:
+                                permissions.add(privilege)
+
+        except Exception as e:
+            logger.warning("Could not check user permissions for %s: %s", username, e)
+            # 安全默认：权限检查失败时拒绝访问
+            pass
+
+        # 缓存权限信息
+        self._permission_cache[cache_key] = permissions
+        return permissions
+
+    def _get_external_volume_permissions(self, volume_name: str) -> set[str]:
+        """获取用户对指定External Volume的权限
+
+        Args:
+            volume_name: External Volume名称
+
+        Returns:
+            用户对该Volume的权限集合
+        """
+        cache_key = f"external_volume:{volume_name}"
+
+        if cache_key in self._permission_cache:
+            return self._permission_cache[cache_key]
+
+        permissions = set()
+
+        try:
+            with self._connection.cursor() as cursor:
+                # 使用正确的ClickZetta语法检查Volume权限
+                logger.info("Checking permissions for volume: %s", volume_name)
+                cursor.execute(f"SHOW GRANTS ON VOLUME {volume_name}")
+                grants = cursor.fetchall()
+
+                logger.info("Raw grants result for %s: %s", volume_name, grants)
+
+                # 解析权限结果
+                # 格式: (granted_type, privilege, conditions, granted_on, object_name, granted_to,
+                #       grantee_name, grantor_name, grant_option, granted_time)
+                for grant in grants:
+                    logger.info("Processing grant: %s", grant)
+                    if len(grant) >= 5:
+                        granted_type = grant[0]
+                        privilege = grant[1].upper()
+                        granted_on = grant[3]
+                        object_name = grant[4]
+
+                        logger.info(
+                            "Grant details - type: %s, privilege: %s, granted_on: %s, object_name: %s",
+                            granted_type,
+                            privilege,
+                            granted_on,
+                            object_name,
+                        )
+
+                        # 检查是否是对该Volume的权限或者是层级权限
+                        if (
+                            granted_type == "PRIVILEGE" and granted_on == "VOLUME" and object_name.endswith(volume_name)
+                        ) or (granted_type == "OBJECT_HIERARCHY" and granted_on == "VOLUME"):
+                            logger.info("Matching grant found for %s", volume_name)
+
+                            if "READ" in privilege:
+                                permissions.add("read")
+                                logger.info("Added READ permission for %s", volume_name)
+                            if "WRITE" in privilege:
+                                permissions.add("write")
+                                logger.info("Added WRITE permission for %s", volume_name)
+                            if "ALTER" in privilege:
+                                permissions.add("alter")
+                                logger.info("Added ALTER permission for %s", volume_name)
+                            if privilege == "ALL":
+                                permissions.update(["read", "write", "alter"])
+                                logger.info("Added ALL permissions for %s", volume_name)
+
+                logger.info("Final permissions for %s: %s", volume_name, permissions)
+
+                # 如果没有找到明确的权限，尝试查看Volume列表来验证基本权限
+                if not permissions:
+                    try:
+                        cursor.execute("SHOW VOLUMES")
+                        volumes = cursor.fetchall()
+                        for volume in volumes:
+                            if len(volume) > 0 and volume[0] == volume_name:
+                                permissions.add("read")  # 至少有读权限
+                                logger.debug("Volume %s found in SHOW VOLUMES, assuming read permission", volume_name)
+                                break
+                    except Exception:
+                        logger.debug("Cannot access volume %s, no basic permission", volume_name)
+
+        except Exception as e:
+            logger.warning("Could not check external volume permissions for %s: %s", volume_name, e)
+            # 在权限检查失败时，尝试基本的Volume访问验证
+            try:
+                with self._connection.cursor() as cursor:
+                    cursor.execute("SHOW VOLUMES")
+                    volumes = cursor.fetchall()
+                    for volume in volumes:
+                        if len(volume) > 0 and volume[0] == volume_name:
+                            logger.info("Basic volume access verified for %s", volume_name)
+                            permissions.add("read")
+                            permissions.add("write")  # 假设有写权限
+                            break
+            except Exception as basic_e:
+                logger.warning("Basic volume access check failed for %s: %s", volume_name, basic_e)
+                # 最后的备选方案：假设有基本权限
+                permissions.add("read")
+
+        # 缓存权限信息
+        self._permission_cache[cache_key] = permissions
+        return permissions
+
+    def clear_permission_cache(self):
+        """清空权限缓存"""
+        self._permission_cache.clear()
+        logger.debug("Permission cache cleared")
+
+    def get_permission_summary(self, dataset_id: Optional[str] = None) -> dict[str, bool]:
+        """获取权限摘要
+
+        Args:
+            dataset_id: 数据集ID (用于table volume)
+
+        Returns:
+            权限摘要字典
+        """
+        summary = {}
+
+        for operation in VolumePermission:
+            summary[operation.name.lower()] = self.check_permission(operation, dataset_id)
+
+        return summary
+
+    def check_inherited_permission(self, file_path: str, operation: VolumePermission) -> bool:
+        """检查文件路径的权限继承
+
+        Args:
+            file_path: 文件路径
+            operation: 要执行的操作
+
+        Returns:
+            True if user has permission, False otherwise
+        """
+        try:
+            # 解析文件路径
+            path_parts = file_path.strip("/").split("/")
+
+            if not path_parts:
+                logger.warning("Invalid file path for permission inheritance check")
+                return False
+
+            # 对于Table Volume，第一层是dataset_id
+            if self._volume_type == "table":
+                if len(path_parts) < 1:
+                    return False
+
+                dataset_id = path_parts[0]
+
+                # 检查对dataset的权限
+                has_dataset_permission = self.check_permission(operation, dataset_id)
+
+                if not has_dataset_permission:
+                    logger.debug("Permission denied for dataset %s", dataset_id)
+                    return False
+
+                # 检查路径遍历攻击
+                if self._contains_path_traversal(file_path):
+                    logger.warning("Path traversal attack detected: %s", file_path)
+                    return False
+
+                # 检查是否访问敏感目录
+                if self._is_sensitive_path(file_path):
+                    logger.warning("Access to sensitive path denied: %s", file_path)
+                    return False
+
+                logger.debug("Permission inherited for path %s", file_path)
+                return True
+
+            elif self._volume_type == "user":
+                # User Volume的权限继承
+                current_user = self._get_current_username()
+
+                # 检查是否试图访问其他用户的目录
+                if len(path_parts) > 1 and path_parts[0] != current_user:
+                    logger.warning("User %s attempted to access %s's directory", current_user, path_parts[0])
+                    return False
+
+                # 检查基本权限
+                return self.check_permission(operation)
+
+            elif self._volume_type == "external":
+                # External Volume的权限继承
+                # 检查对External Volume的权限
+                return self.check_permission(operation)
+
+            else:
+                logger.warning("Unknown volume type for permission inheritance: %s", self._volume_type)
+                return False
+
+        except Exception as e:
+            logger.exception("Permission inheritance check failed")
+            return False
+
+    def _contains_path_traversal(self, file_path: str) -> bool:
+        """检查路径是否包含路径遍历攻击"""
+        # 检查常见的路径遍历模式
+        traversal_patterns = [
+            "../",
+            "..\\",
+            "..%2f",
+            "..%2F",
+            "..%5c",
+            "..%5C",
+            "%2e%2e%2f",
+            "%2e%2e%5c",
+            "....//",
+            "....\\\\",
+        ]
+
+        file_path_lower = file_path.lower()
+
+        for pattern in traversal_patterns:
+            if pattern in file_path_lower:
+                return True
+
+        # 检查绝对路径
+        if file_path.startswith("/") or file_path.startswith("\\"):
+            return True
+
+        # 检查Windows驱动器路径
+        if len(file_path) >= 2 and file_path[1] == ":":
+            return True
+
+        return False
+
+    def _is_sensitive_path(self, file_path: str) -> bool:
+        """检查路径是否为敏感路径"""
+        sensitive_patterns = [
+            "passwd",
+            "shadow",
+            "hosts",
+            "config",
+            "secrets",
+            "private",
+            "key",
+            "certificate",
+            "cert",
+            "ssl",
+            "database",
+            "backup",
+            "dump",
+            "log",
+            "tmp",
+        ]
+
+        file_path_lower = file_path.lower()
+
+        return any(pattern in file_path_lower for pattern in sensitive_patterns)
+
+    def validate_operation(self, operation: str, dataset_id: Optional[str] = None) -> bool:
+        """验证操作权限
+
+        Args:
+            operation: 操作名称 (save|load|exists|delete|scan)
+            dataset_id: 数据集ID
+
+        Returns:
+            True if operation is allowed, False otherwise
+        """
+        operation_mapping = {
+            "save": VolumePermission.WRITE,
+            "load": VolumePermission.READ,
+            "load_once": VolumePermission.READ,
+            "load_stream": VolumePermission.READ,
+            "download": VolumePermission.READ,
+            "exists": VolumePermission.READ,
+            "delete": VolumePermission.DELETE,
+            "scan": VolumePermission.LIST,
+        }
+
+        if operation not in operation_mapping:
+            logger.warning("Unknown operation: %s", operation)
+            return False
+
+        volume_permission = operation_mapping[operation]
+        return self.check_permission(volume_permission, dataset_id)
+
+
+class VolumePermissionError(Exception):
+    """Volume权限错误异常"""
+
+    def __init__(self, message: str, operation: str, volume_type: str, dataset_id: Optional[str] = None):
+        self.operation = operation
+        self.volume_type = volume_type
+        self.dataset_id = dataset_id
+        super().__init__(message)
+
+
+def check_volume_permission(
+    permission_manager: VolumePermissionManager, operation: str, dataset_id: Optional[str] = None
+) -> None:
+    """权限检查装饰器函数
+
+    Args:
+        permission_manager: 权限管理器
+        operation: 操作名称
+        dataset_id: 数据集ID
+
+    Raises:
+        VolumePermissionError: 如果没有权限
+    """
+    if not permission_manager.validate_operation(operation, dataset_id):
+        error_message = f"Permission denied for operation '{operation}' on {permission_manager._volume_type} volume"
+        if dataset_id:
+            error_message += f" (dataset: {dataset_id})"
+
+        raise VolumePermissionError(
+            error_message,
+            operation=operation,
+            volume_type=permission_manager._volume_type or "unknown",
+            dataset_id=dataset_id,
+        )

api/extensions/storage/storage_type.py

@@ -5,6 +5,7 @@ class StorageType(StrEnum):
     ALIYUN_OSS = "aliyun-oss"
     AZURE_BLOB = "azure-blob"
     BAIDU_OBS = "baidu-obs"
+    CLICKZETTA_VOLUME = "clickzetta-volume"
     GOOGLE_STORAGE = "google-storage"
     HUAWEI_OBS = "huawei-obs"
     LOCAL = "local"

api/fields/app_fields.py

@@ -59,6 +59,8 @@ model_config_fields = {
     "updated_at": TimestampField,
 }
 
+tag_fields = {"id": fields.String, "name": fields.String, "type": fields.String}
+
 app_detail_fields = {
     "id": fields.String,
     "name": fields.String,
@@ -77,6 +79,7 @@ app_detail_fields = {
     "updated_by": fields.String,
     "updated_at": TimestampField,
     "access_mode": fields.String,
+    "tags": fields.List(fields.Nested(tag_fields)),
 }
 
 prompt_config_fields = {
@@ -92,8 +95,6 @@ model_config_partial_fields = {
     "updated_at": TimestampField,
 }
 
-tag_fields = {"id": fields.String, "name": fields.String, "type": fields.String}
-
 app_partial_fields = {
     "id": fields.String,
     "name": fields.String,
@@ -185,7 +186,6 @@ app_detail_fields_with_site = {
     "enable_api": fields.Boolean,
     "model_config": fields.Nested(model_config_fields, attribute="app_model_config", allow_null=True),
     "workflow": fields.Nested(workflow_partial_fields, allow_null=True),
-    "site": fields.Nested(site_fields),
     "api_base_url": fields.String,
     "use_icon_as_answer_icon": fields.Boolean,
     "max_active_requests": fields.Integer,
@@ -195,6 +195,8 @@ app_detail_fields_with_site = {
     "updated_at": TimestampField,
     "deleted_tools": fields.List(fields.Nested(deleted_tool_fields)),
     "access_mode": fields.String,
+    "tags": fields.List(fields.Nested(tag_fields)),
+    "site": fields.Nested(site_fields),
 }
 
 

api/libs/rsa.py

@@ -1,5 +1,4 @@
 import hashlib
-import os
 from typing import Union
 
 from Crypto.Cipher import AES
@@ -18,7 +17,7 @@ def generate_key_pair(tenant_id: str) -> str:
     pem_private = private_key.export_key()
     pem_public = public_key.export_key()
 
-    filepath = os.path.join("privkeys", tenant_id, "private.pem")
+    filepath = f"privkeys/{tenant_id}/private.pem"
 
     storage.save(filepath, pem_private)
 
@@ -48,7 +47,7 @@ def encrypt(text: str, public_key: Union[str, bytes]) -> bytes:
 
 
 def get_decrypt_decoding(tenant_id: str) -> tuple[RSA.RsaKey, object]:
-    filepath = os.path.join("privkeys", tenant_id, "private.pem")
+    filepath = f"privkeys/{tenant_id}/private.pem"
 
     cache_key = f"tenant_privkey:{hashlib.sha3_256(filepath.encode()).hexdigest()}"
     private_key = redis_client.get(cache_key)

api/migrations/versions/2025_07_24_1450-532b3f888abf_manual_dataset_field_update.py

@@ -0,0 +1,25 @@
+"""manual dataset field update
+
+Revision ID: 532b3f888abf
+Revises: 8bcc02c9bd07
+Create Date: 2025-07-24 14:50:48.779833
+
+"""
+from alembic import op
+import models as models
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '532b3f888abf'
+down_revision = '8bcc02c9bd07'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("ALTER TABLE tidb_auth_bindings ALTER COLUMN status SET DEFAULT 'CREATING'::character varying")
+
+
+def downgrade():
+    op.execute("ALTER TABLE tidb_auth_bindings ALTER COLUMN status SET DEFAULT 'CREATING'")

api/models/account.py

@@ -3,6 +3,7 @@ import json
 from datetime import datetime
 from typing import Optional, cast
 
+import sqlalchemy as sa
 from flask_login import UserMixin  # type: ignore
 from sqlalchemy import DateTime, String, func, select
 from sqlalchemy.orm import Mapped, mapped_column, reconstructor
@@ -83,9 +84,9 @@ class AccountStatus(enum.StrEnum):
 
 class Account(UserMixin, Base):
     __tablename__ = "accounts"
-    __table_args__ = (db.PrimaryKeyConstraint("id", name="account_pkey"), db.Index("account_email_idx", "email"))
+    __table_args__ = (sa.PrimaryKeyConstraint("id", name="account_pkey"), sa.Index("account_email_idx", "email"))
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     name: Mapped[str] = mapped_column(String(255))
     email: Mapped[str] = mapped_column(String(255))
     password: Mapped[Optional[str]] = mapped_column(String(255))
@@ -97,7 +98,7 @@ class Account(UserMixin, Base):
     last_login_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
     last_login_ip: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
     last_active_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp(), nullable=False)
-    status: Mapped[str] = mapped_column(String(16), server_default=db.text("'active'::character varying"))
+    status: Mapped[str] = mapped_column(String(16), server_default=sa.text("'active'::character varying"))
     initialized_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
     created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp(), nullable=False)
     updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp(), nullable=False)
@@ -195,14 +196,14 @@ class TenantStatus(enum.StrEnum):
 
 class Tenant(Base):
     __tablename__ = "tenants"
-    __table_args__ = (db.PrimaryKeyConstraint("id", name="tenant_pkey"),)
+    __table_args__ = (sa.PrimaryKeyConstraint("id", name="tenant_pkey"),)
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     name: Mapped[str] = mapped_column(String(255))
-    encrypt_public_key = db.Column(db.Text)
-    plan: Mapped[str] = mapped_column(String(255), server_default=db.text("'basic'::character varying"))
-    status: Mapped[str] = mapped_column(String(255), server_default=db.text("'normal'::character varying"))
-    custom_config: Mapped[Optional[str]] = mapped_column(db.Text)
+    encrypt_public_key = db.Column(sa.Text)
+    plan: Mapped[str] = mapped_column(String(255), server_default=sa.text("'basic'::character varying"))
+    status: Mapped[str] = mapped_column(String(255), server_default=sa.text("'normal'::character varying"))
+    custom_config: Mapped[Optional[str]] = mapped_column(sa.Text)
     created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp(), nullable=False)
     updated_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp())
 
@@ -225,16 +226,16 @@ class Tenant(Base):
 class TenantAccountJoin(Base):
     __tablename__ = "tenant_account_joins"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tenant_account_join_pkey"),
-        db.Index("tenant_account_join_account_id_idx", "account_id"),
-        db.Index("tenant_account_join_tenant_id_idx", "tenant_id"),
-        db.UniqueConstraint("tenant_id", "account_id", name="unique_tenant_account_join"),
+        sa.PrimaryKeyConstraint("id", name="tenant_account_join_pkey"),
+        sa.Index("tenant_account_join_account_id_idx", "account_id"),
+        sa.Index("tenant_account_join_tenant_id_idx", "tenant_id"),
+        sa.UniqueConstraint("tenant_id", "account_id", name="unique_tenant_account_join"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID)
     account_id: Mapped[str] = mapped_column(StringUUID)
-    current: Mapped[bool] = mapped_column(db.Boolean, server_default=db.text("false"))
+    current: Mapped[bool] = mapped_column(sa.Boolean, server_default=sa.text("false"))
     role: Mapped[str] = mapped_column(String(16), server_default="normal")
     invited_by: Mapped[Optional[str]] = mapped_column(StringUUID)
     created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp())
@@ -244,12 +245,12 @@ class TenantAccountJoin(Base):
 class AccountIntegrate(Base):
     __tablename__ = "account_integrates"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="account_integrate_pkey"),
-        db.UniqueConstraint("account_id", "provider", name="unique_account_provider"),
-        db.UniqueConstraint("provider", "open_id", name="unique_provider_open_id"),
+        sa.PrimaryKeyConstraint("id", name="account_integrate_pkey"),
+        sa.UniqueConstraint("account_id", "provider", name="unique_account_provider"),
+        sa.UniqueConstraint("provider", "open_id", name="unique_provider_open_id"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     account_id: Mapped[str] = mapped_column(StringUUID)
     provider: Mapped[str] = mapped_column(String(16))
     open_id: Mapped[str] = mapped_column(String(255))
@@ -261,20 +262,20 @@ class AccountIntegrate(Base):
 class InvitationCode(Base):
     __tablename__ = "invitation_codes"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="invitation_code_pkey"),
-        db.Index("invitation_codes_batch_idx", "batch"),
-        db.Index("invitation_codes_code_idx", "code", "status"),
+        sa.PrimaryKeyConstraint("id", name="invitation_code_pkey"),
+        sa.Index("invitation_codes_batch_idx", "batch"),
+        sa.Index("invitation_codes_code_idx", "code", "status"),
     )
 
-    id: Mapped[int] = mapped_column(db.Integer)
+    id: Mapped[int] = mapped_column(sa.Integer)
     batch: Mapped[str] = mapped_column(String(255))
     code: Mapped[str] = mapped_column(String(32))
-    status: Mapped[str] = mapped_column(String(16), server_default=db.text("'unused'::character varying"))
+    status: Mapped[str] = mapped_column(String(16), server_default=sa.text("'unused'::character varying"))
     used_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
     used_by_tenant_id: Mapped[Optional[str]] = mapped_column(StringUUID)
     used_by_account_id: Mapped[Optional[str]] = mapped_column(StringUUID)
     deprecated_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
-    created_at: Mapped[datetime] = mapped_column(DateTime, server_default=db.text("CURRENT_TIMESTAMP(0)"))
+    created_at: Mapped[datetime] = mapped_column(DateTime, server_default=sa.text("CURRENT_TIMESTAMP(0)"))
 
 
 class TenantPluginPermission(Base):
@@ -290,11 +291,11 @@ class TenantPluginPermission(Base):
 
     __tablename__ = "account_plugin_permissions"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="account_plugin_permission_pkey"),
-        db.UniqueConstraint("tenant_id", name="unique_tenant_plugin"),
+        sa.PrimaryKeyConstraint("id", name="account_plugin_permission_pkey"),
+        sa.UniqueConstraint("tenant_id", name="unique_tenant_plugin"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     install_permission: Mapped[InstallPermission] = mapped_column(String(16), nullable=False, server_default="everyone")
     debug_permission: Mapped[DebugPermission] = mapped_column(String(16), nullable=False, server_default="noone")
@@ -313,16 +314,16 @@ class TenantPluginAutoUpgradeStrategy(Base):
 
     __tablename__ = "tenant_plugin_auto_upgrade_strategies"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tenant_plugin_auto_upgrade_strategy_pkey"),
-        db.UniqueConstraint("tenant_id", name="unique_tenant_plugin_auto_upgrade_strategy"),
+        sa.PrimaryKeyConstraint("id", name="tenant_plugin_auto_upgrade_strategy_pkey"),
+        sa.UniqueConstraint("tenant_id", name="unique_tenant_plugin_auto_upgrade_strategy"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     strategy_setting: Mapped[StrategySetting] = mapped_column(String(16), nullable=False, server_default="fix_only")
-    upgrade_time_of_day: Mapped[int] = mapped_column(db.Integer, nullable=False, default=0)  # seconds of the day
+    upgrade_time_of_day: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)  # seconds of the day
     upgrade_mode: Mapped[UpgradeMode] = mapped_column(String(16), nullable=False, server_default="exclude")
-    exclude_plugins: Mapped[list[str]] = mapped_column(db.ARRAY(String(255)), nullable=False)  # plugin_id (author/name)
-    include_plugins: Mapped[list[str]] = mapped_column(db.ARRAY(String(255)), nullable=False)  # plugin_id (author/name)
+    exclude_plugins: Mapped[list[str]] = mapped_column(sa.ARRAY(String(255)), nullable=False)  # plugin_id (author/name)
+    include_plugins: Mapped[list[str]] = mapped_column(sa.ARRAY(String(255)), nullable=False)  # plugin_id (author/name)
     created_at = db.Column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at = db.Column(DateTime, nullable=False, server_default=func.current_timestamp())

api/models/api_based_extension.py

@@ -1,11 +1,11 @@
 import enum
 from datetime import datetime
 
+import sqlalchemy as sa
 from sqlalchemy import DateTime, String, Text, func
 from sqlalchemy.orm import Mapped, mapped_column
 
 from .base import Base
-from .engine import db
 from .types import StringUUID
 
 
@@ -19,11 +19,11 @@ class APIBasedExtensionPoint(enum.Enum):
 class APIBasedExtension(Base):
     __tablename__ = "api_based_extensions"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="api_based_extension_pkey"),
-        db.Index("api_based_extension_tenant_idx", "tenant_id"),
+        sa.PrimaryKeyConstraint("id", name="api_based_extension_pkey"),
+        sa.Index("api_based_extension_tenant_idx", "tenant_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     name: Mapped[str] = mapped_column(String(255), nullable=False)
     api_endpoint: Mapped[str] = mapped_column(String(255), nullable=False)

api/models/dataset.py

@@ -12,6 +12,7 @@ from datetime import datetime
 from json import JSONDecodeError
 from typing import Any, Optional, cast
 
+import sqlalchemy as sa
 from sqlalchemy import DateTime, String, func, select
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column
@@ -38,23 +39,23 @@ class DatasetPermissionEnum(enum.StrEnum):
 class Dataset(Base):
     __tablename__ = "datasets"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_pkey"),
-        db.Index("dataset_tenant_idx", "tenant_id"),
-        db.Index("retrieval_model_idx", "retrieval_model", postgresql_using="gin"),
+        sa.PrimaryKeyConstraint("id", name="dataset_pkey"),
+        sa.Index("dataset_tenant_idx", "tenant_id"),
+        sa.Index("retrieval_model_idx", "retrieval_model", postgresql_using="gin"),
     )
 
     INDEXING_TECHNIQUE_LIST = ["high_quality", "economy", None]
     PROVIDER_LIST = ["vendor", "external", None]
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID)
     name: Mapped[str] = mapped_column(String(255))
-    description = mapped_column(db.Text, nullable=True)
-    provider: Mapped[str] = mapped_column(String(255), server_default=db.text("'vendor'::character varying"))
-    permission: Mapped[str] = mapped_column(String(255), server_default=db.text("'only_me'::character varying"))
+    description = mapped_column(sa.Text, nullable=True)
+    provider: Mapped[str] = mapped_column(String(255), server_default=sa.text("'vendor'::character varying"))
+    permission: Mapped[str] = mapped_column(String(255), server_default=sa.text("'only_me'::character varying"))
     data_source_type = mapped_column(String(255))
     indexing_technique: Mapped[Optional[str]] = mapped_column(String(255))
-    index_struct = mapped_column(db.Text, nullable=True)
+    index_struct = mapped_column(sa.Text, nullable=True)
     created_by = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_by = mapped_column(StringUUID, nullable=True)
@@ -63,7 +64,7 @@ class Dataset(Base):
     embedding_model_provider = db.Column(String(255), nullable=True)  # TODO: mapped_column
     collection_binding_id = mapped_column(StringUUID, nullable=True)
     retrieval_model = mapped_column(JSONB, nullable=True)
-    built_in_field_enabled: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    built_in_field_enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
 
     @property
     def dataset_keyword_table(self):
@@ -262,14 +263,14 @@ class Dataset(Base):
 class DatasetProcessRule(Base):
     __tablename__ = "dataset_process_rules"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_process_rule_pkey"),
-        db.Index("dataset_process_rule_dataset_id_idx", "dataset_id"),
+        sa.PrimaryKeyConstraint("id", name="dataset_process_rule_pkey"),
+        sa.Index("dataset_process_rule_dataset_id_idx", "dataset_id"),
     )
 
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     dataset_id = mapped_column(StringUUID, nullable=False)
-    mode = mapped_column(String(255), nullable=False, server_default=db.text("'automatic'::character varying"))
-    rules = mapped_column(db.Text, nullable=True)
+    mode = mapped_column(String(255), nullable=False, server_default=sa.text("'automatic'::character varying"))
+    rules = mapped_column(sa.Text, nullable=True)
     created_by = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
 
@@ -302,20 +303,20 @@ class DatasetProcessRule(Base):
 class Document(Base):
     __tablename__ = "documents"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="document_pkey"),
-        db.Index("document_dataset_id_idx", "dataset_id"),
-        db.Index("document_is_paused_idx", "is_paused"),
-        db.Index("document_tenant_idx", "tenant_id"),
-        db.Index("document_metadata_idx", "doc_metadata", postgresql_using="gin"),
+        sa.PrimaryKeyConstraint("id", name="document_pkey"),
+        sa.Index("document_dataset_id_idx", "dataset_id"),
+        sa.Index("document_is_paused_idx", "is_paused"),
+        sa.Index("document_tenant_idx", "tenant_id"),
+        sa.Index("document_metadata_idx", "doc_metadata", postgresql_using="gin"),
     )
 
     # initial fields
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     dataset_id = mapped_column(StringUUID, nullable=False)
-    position: Mapped[int] = mapped_column(db.Integer, nullable=False)
+    position: Mapped[int] = mapped_column(sa.Integer, nullable=False)
     data_source_type: Mapped[str] = mapped_column(String(255), nullable=False)
-    data_source_info = mapped_column(db.Text, nullable=True)
+    data_source_info = mapped_column(sa.Text, nullable=True)
     dataset_process_rule_id = mapped_column(StringUUID, nullable=True)
     batch: Mapped[str] = mapped_column(String(255), nullable=False)
     name: Mapped[str] = mapped_column(String(255), nullable=False)
@@ -328,8 +329,8 @@ class Document(Base):
     processing_started_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     # parsing
-    file_id = mapped_column(db.Text, nullable=True)
-    word_count: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)  # TODO: make this not nullable
+    file_id = mapped_column(sa.Text, nullable=True)
+    word_count: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)  # TODO: make this not nullable
     parsing_completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     # cleaning
@@ -339,32 +340,32 @@ class Document(Base):
     splitting_completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     # indexing
-    tokens: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)
-    indexing_latency: Mapped[Optional[float]] = mapped_column(db.Float, nullable=True)
+    tokens: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
+    indexing_latency: Mapped[Optional[float]] = mapped_column(sa.Float, nullable=True)
     completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     # pause
-    is_paused: Mapped[Optional[bool]] = mapped_column(db.Boolean, nullable=True, server_default=db.text("false"))
+    is_paused: Mapped[Optional[bool]] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"))
     paused_by = mapped_column(StringUUID, nullable=True)
     paused_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     # error
-    error = mapped_column(db.Text, nullable=True)
+    error = mapped_column(sa.Text, nullable=True)
     stopped_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     # basic fields
-    indexing_status = mapped_column(String(255), nullable=False, server_default=db.text("'waiting'::character varying"))
-    enabled: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("true"))
+    indexing_status = mapped_column(String(255), nullable=False, server_default=sa.text("'waiting'::character varying"))
+    enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
     disabled_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
     disabled_by = mapped_column(StringUUID, nullable=True)
-    archived: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    archived: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
     archived_reason = mapped_column(String(255), nullable=True)
     archived_by = mapped_column(StringUUID, nullable=True)
     archived_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
     updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     doc_type = mapped_column(String(40), nullable=True)
     doc_metadata = mapped_column(JSONB, nullable=True)
-    doc_form = mapped_column(String(255), nullable=False, server_default=db.text("'text_model'::character varying"))
+    doc_form = mapped_column(String(255), nullable=False, server_default=sa.text("'text_model'::character varying"))
     doc_language = mapped_column(String(255), nullable=True)
 
     DATA_SOURCES = ["upload_file", "notion_import", "website_crawl"]
@@ -643,44 +644,44 @@ class Document(Base):
 class DocumentSegment(Base):
     __tablename__ = "document_segments"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="document_segment_pkey"),
-        db.Index("document_segment_dataset_id_idx", "dataset_id"),
-        db.Index("document_segment_document_id_idx", "document_id"),
-        db.Index("document_segment_tenant_dataset_idx", "dataset_id", "tenant_id"),
-        db.Index("document_segment_tenant_document_idx", "document_id", "tenant_id"),
-        db.Index("document_segment_node_dataset_idx", "index_node_id", "dataset_id"),
-        db.Index("document_segment_tenant_idx", "tenant_id"),
+        sa.PrimaryKeyConstraint("id", name="document_segment_pkey"),
+        sa.Index("document_segment_dataset_id_idx", "dataset_id"),
+        sa.Index("document_segment_document_id_idx", "document_id"),
+        sa.Index("document_segment_tenant_dataset_idx", "dataset_id", "tenant_id"),
+        sa.Index("document_segment_tenant_document_idx", "document_id", "tenant_id"),
+        sa.Index("document_segment_node_dataset_idx", "index_node_id", "dataset_id"),
+        sa.Index("document_segment_tenant_idx", "tenant_id"),
     )
 
     # initial fields
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     dataset_id = mapped_column(StringUUID, nullable=False)
     document_id = mapped_column(StringUUID, nullable=False)
     position: Mapped[int]
-    content = mapped_column(db.Text, nullable=False)
-    answer = mapped_column(db.Text, nullable=True)
+    content = mapped_column(sa.Text, nullable=False)
+    answer = mapped_column(sa.Text, nullable=True)
     word_count: Mapped[int]
     tokens: Mapped[int]
 
     # indexing fields
-    keywords = mapped_column(db.JSON, nullable=True)
+    keywords = mapped_column(sa.JSON, nullable=True)
     index_node_id = mapped_column(String(255), nullable=True)
     index_node_hash = mapped_column(String(255), nullable=True)
 
     # basic fields
-    hit_count: Mapped[int] = mapped_column(db.Integer, nullable=False, default=0)
-    enabled: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("true"))
+    hit_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)
+    enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
     disabled_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
     disabled_by = mapped_column(StringUUID, nullable=True)
-    status: Mapped[str] = mapped_column(String(255), server_default=db.text("'waiting'::character varying"))
+    status: Mapped[str] = mapped_column(String(255), server_default=sa.text("'waiting'::character varying"))
     created_by = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_by = mapped_column(StringUUID, nullable=True)
     updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     indexing_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
     completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
-    error = mapped_column(db.Text, nullable=True)
+    error = mapped_column(sa.Text, nullable=True)
     stopped_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     @property
@@ -794,36 +795,36 @@ class DocumentSegment(Base):
 class ChildChunk(Base):
     __tablename__ = "child_chunks"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="child_chunk_pkey"),
-        db.Index("child_chunk_dataset_id_idx", "tenant_id", "dataset_id", "document_id", "segment_id", "index_node_id"),
-        db.Index("child_chunks_node_idx", "index_node_id", "dataset_id"),
-        db.Index("child_chunks_segment_idx", "segment_id"),
+        sa.PrimaryKeyConstraint("id", name="child_chunk_pkey"),
+        sa.Index("child_chunk_dataset_id_idx", "tenant_id", "dataset_id", "document_id", "segment_id", "index_node_id"),
+        sa.Index("child_chunks_node_idx", "index_node_id", "dataset_id"),
+        sa.Index("child_chunks_segment_idx", "segment_id"),
     )
 
     # initial fields
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     dataset_id = mapped_column(StringUUID, nullable=False)
     document_id = mapped_column(StringUUID, nullable=False)
     segment_id = mapped_column(StringUUID, nullable=False)
-    position: Mapped[int] = mapped_column(db.Integer, nullable=False)
-    content = mapped_column(db.Text, nullable=False)
-    word_count: Mapped[int] = mapped_column(db.Integer, nullable=False)
+    position: Mapped[int] = mapped_column(sa.Integer, nullable=False)
+    content = mapped_column(sa.Text, nullable=False)
+    word_count: Mapped[int] = mapped_column(sa.Integer, nullable=False)
     # indexing fields
     index_node_id = mapped_column(String(255), nullable=True)
     index_node_hash = mapped_column(String(255), nullable=True)
-    type = mapped_column(String(255), nullable=False, server_default=db.text("'automatic'::character varying"))
+    type = mapped_column(String(255), nullable=False, server_default=sa.text("'automatic'::character varying"))
     created_by = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(
-        DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
     updated_by = mapped_column(StringUUID, nullable=True)
     updated_at: Mapped[datetime] = mapped_column(
-        DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
     indexing_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
     completed_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
-    error = mapped_column(db.Text, nullable=True)
+    error = mapped_column(sa.Text, nullable=True)
 
     @property
     def dataset(self):
@@ -841,11 +842,11 @@ class ChildChunk(Base):
 class AppDatasetJoin(Base):
     __tablename__ = "app_dataset_joins"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="app_dataset_join_pkey"),
-        db.Index("app_dataset_join_app_dataset_idx", "dataset_id", "app_id"),
+        sa.PrimaryKeyConstraint("id", name="app_dataset_join_pkey"),
+        sa.Index("app_dataset_join_app_dataset_idx", "dataset_id", "app_id"),
     )
 
-    id = mapped_column(StringUUID, primary_key=True, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, primary_key=True, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     dataset_id = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=db.func.current_timestamp())
@@ -858,13 +859,13 @@ class AppDatasetJoin(Base):
 class DatasetQuery(Base):
     __tablename__ = "dataset_queries"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_query_pkey"),
-        db.Index("dataset_query_dataset_id_idx", "dataset_id"),
+        sa.PrimaryKeyConstraint("id", name="dataset_query_pkey"),
+        sa.Index("dataset_query_dataset_id_idx", "dataset_id"),
     )
 
-    id = mapped_column(StringUUID, primary_key=True, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, primary_key=True, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     dataset_id = mapped_column(StringUUID, nullable=False)
-    content = mapped_column(db.Text, nullable=False)
+    content = mapped_column(sa.Text, nullable=False)
     source: Mapped[str] = mapped_column(String(255), nullable=False)
     source_app_id = mapped_column(StringUUID, nullable=True)
     created_by_role = mapped_column(String, nullable=False)
@@ -875,15 +876,15 @@ class DatasetQuery(Base):
 class DatasetKeywordTable(Base):
     __tablename__ = "dataset_keyword_tables"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_keyword_table_pkey"),
-        db.Index("dataset_keyword_table_dataset_id_idx", "dataset_id"),
+        sa.PrimaryKeyConstraint("id", name="dataset_keyword_table_pkey"),
+        sa.Index("dataset_keyword_table_dataset_id_idx", "dataset_id"),
     )
 
-    id = mapped_column(StringUUID, primary_key=True, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
     dataset_id = mapped_column(StringUUID, nullable=False, unique=True)
-    keyword_table = mapped_column(db.Text, nullable=False)
+    keyword_table = mapped_column(sa.Text, nullable=False)
     data_source_type = mapped_column(
-        String(255), nullable=False, server_default=db.text("'database'::character varying")
+        String(255), nullable=False, server_default=sa.text("'database'::character varying")
     )
 
     @property
@@ -920,19 +921,19 @@ class DatasetKeywordTable(Base):
 class Embedding(Base):
     __tablename__ = "embeddings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="embedding_pkey"),
-        db.UniqueConstraint("model_name", "hash", "provider_name", name="embedding_hash_idx"),
-        db.Index("created_at_idx", "created_at"),
+        sa.PrimaryKeyConstraint("id", name="embedding_pkey"),
+        sa.UniqueConstraint("model_name", "hash", "provider_name", name="embedding_hash_idx"),
+        sa.Index("created_at_idx", "created_at"),
     )
 
-    id = mapped_column(StringUUID, primary_key=True, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
     model_name = mapped_column(
-        String(255), nullable=False, server_default=db.text("'text-embedding-ada-002'::character varying")
+        String(255), nullable=False, server_default=sa.text("'text-embedding-ada-002'::character varying")
     )
     hash = mapped_column(String(64), nullable=False)
-    embedding = mapped_column(db.LargeBinary, nullable=False)
+    embedding = mapped_column(sa.LargeBinary, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
-    provider_name = mapped_column(String(255), nullable=False, server_default=db.text("''::character varying"))
+    provider_name = mapped_column(String(255), nullable=False, server_default=sa.text("''::character varying"))
 
     def set_embedding(self, embedding_data: list[float]):
         self.embedding = pickle.dumps(embedding_data, protocol=pickle.HIGHEST_PROTOCOL)
@@ -944,14 +945,14 @@ class Embedding(Base):
 class DatasetCollectionBinding(Base):
     __tablename__ = "dataset_collection_bindings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_collection_bindings_pkey"),
-        db.Index("provider_model_name_idx", "provider_name", "model_name"),
+        sa.PrimaryKeyConstraint("id", name="dataset_collection_bindings_pkey"),
+        sa.Index("provider_model_name_idx", "provider_name", "model_name"),
     )
 
-    id = mapped_column(StringUUID, primary_key=True, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
     provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
     model_name: Mapped[str] = mapped_column(String(255), nullable=False)
-    type = mapped_column(String(40), server_default=db.text("'dataset'::character varying"), nullable=False)
+    type = mapped_column(String(40), server_default=sa.text("'dataset'::character varying"), nullable=False)
     collection_name = mapped_column(String(64), nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
 
@@ -959,18 +960,18 @@ class DatasetCollectionBinding(Base):
 class TidbAuthBinding(Base):
     __tablename__ = "tidb_auth_bindings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tidb_auth_bindings_pkey"),
-        db.Index("tidb_auth_bindings_tenant_idx", "tenant_id"),
-        db.Index("tidb_auth_bindings_active_idx", "active"),
-        db.Index("tidb_auth_bindings_created_at_idx", "created_at"),
-        db.Index("tidb_auth_bindings_status_idx", "status"),
+        sa.PrimaryKeyConstraint("id", name="tidb_auth_bindings_pkey"),
+        sa.Index("tidb_auth_bindings_tenant_idx", "tenant_id"),
+        sa.Index("tidb_auth_bindings_active_idx", "active"),
+        sa.Index("tidb_auth_bindings_created_at_idx", "created_at"),
+        sa.Index("tidb_auth_bindings_status_idx", "status"),
     )
-    id = mapped_column(StringUUID, primary_key=True, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=True)
     cluster_id: Mapped[str] = mapped_column(String(255), nullable=False)
     cluster_name: Mapped[str] = mapped_column(String(255), nullable=False)
-    active: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
-    status = mapped_column(String(255), nullable=False, server_default=db.text("CREATING"))
+    active: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=db.text("false"))
+    status = mapped_column(String(255), nullable=False, server_default=db.text("'CREATING'::character varying"))
     account: Mapped[str] = mapped_column(String(255), nullable=False)
     password: Mapped[str] = mapped_column(String(255), nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
@@ -979,10 +980,10 @@ class TidbAuthBinding(Base):
 class Whitelist(Base):
     __tablename__ = "whitelists"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="whitelists_pkey"),
-        db.Index("whitelists_tenant_idx", "tenant_id"),
+        sa.PrimaryKeyConstraint("id", name="whitelists_pkey"),
+        sa.Index("whitelists_tenant_idx", "tenant_id"),
     )
-    id = mapped_column(StringUUID, primary_key=True, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=True)
     category: Mapped[str] = mapped_column(String(255), nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
@@ -991,33 +992,33 @@ class Whitelist(Base):
 class DatasetPermission(Base):
     __tablename__ = "dataset_permissions"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_permission_pkey"),
-        db.Index("idx_dataset_permissions_dataset_id", "dataset_id"),
-        db.Index("idx_dataset_permissions_account_id", "account_id"),
-        db.Index("idx_dataset_permissions_tenant_id", "tenant_id"),
+        sa.PrimaryKeyConstraint("id", name="dataset_permission_pkey"),
+        sa.Index("idx_dataset_permissions_dataset_id", "dataset_id"),
+        sa.Index("idx_dataset_permissions_account_id", "account_id"),
+        sa.Index("idx_dataset_permissions_tenant_id", "tenant_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"), primary_key=True)
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"), primary_key=True)
     dataset_id = mapped_column(StringUUID, nullable=False)
     account_id = mapped_column(StringUUID, nullable=False)
     tenant_id = mapped_column(StringUUID, nullable=False)
-    has_permission: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("true"))
+    has_permission: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
 
 
 class ExternalKnowledgeApis(Base):
     __tablename__ = "external_knowledge_apis"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="external_knowledge_apis_pkey"),
-        db.Index("external_knowledge_apis_tenant_idx", "tenant_id"),
-        db.Index("external_knowledge_apis_name_idx", "name"),
+        sa.PrimaryKeyConstraint("id", name="external_knowledge_apis_pkey"),
+        sa.Index("external_knowledge_apis_tenant_idx", "tenant_id"),
+        sa.Index("external_knowledge_apis_name_idx", "name"),
     )
 
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     name: Mapped[str] = mapped_column(String(255), nullable=False)
     description: Mapped[str] = mapped_column(String(255), nullable=False)
     tenant_id = mapped_column(StringUUID, nullable=False)
-    settings = mapped_column(db.Text, nullable=True)
+    settings = mapped_column(sa.Text, nullable=True)
     created_by = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_by = mapped_column(StringUUID, nullable=True)
@@ -1061,18 +1062,18 @@ class ExternalKnowledgeApis(Base):
 class ExternalKnowledgeBindings(Base):
     __tablename__ = "external_knowledge_bindings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="external_knowledge_bindings_pkey"),
-        db.Index("external_knowledge_bindings_tenant_idx", "tenant_id"),
-        db.Index("external_knowledge_bindings_dataset_idx", "dataset_id"),
-        db.Index("external_knowledge_bindings_external_knowledge_idx", "external_knowledge_id"),
-        db.Index("external_knowledge_bindings_external_knowledge_api_idx", "external_knowledge_api_id"),
+        sa.PrimaryKeyConstraint("id", name="external_knowledge_bindings_pkey"),
+        sa.Index("external_knowledge_bindings_tenant_idx", "tenant_id"),
+        sa.Index("external_knowledge_bindings_dataset_idx", "dataset_id"),
+        sa.Index("external_knowledge_bindings_external_knowledge_idx", "external_knowledge_id"),
+        sa.Index("external_knowledge_bindings_external_knowledge_api_idx", "external_knowledge_api_id"),
     )
 
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     external_knowledge_api_id = mapped_column(StringUUID, nullable=False)
     dataset_id = mapped_column(StringUUID, nullable=False)
-    external_knowledge_id = mapped_column(db.Text, nullable=False)
+    external_knowledge_id = mapped_column(sa.Text, nullable=False)
     created_by = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_by = mapped_column(StringUUID, nullable=True)
@@ -1082,57 +1083,57 @@ class ExternalKnowledgeBindings(Base):
 class DatasetAutoDisableLog(Base):
     __tablename__ = "dataset_auto_disable_logs"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_auto_disable_log_pkey"),
-        db.Index("dataset_auto_disable_log_tenant_idx", "tenant_id"),
-        db.Index("dataset_auto_disable_log_dataset_idx", "dataset_id"),
-        db.Index("dataset_auto_disable_log_created_atx", "created_at"),
+        sa.PrimaryKeyConstraint("id", name="dataset_auto_disable_log_pkey"),
+        sa.Index("dataset_auto_disable_log_tenant_idx", "tenant_id"),
+        sa.Index("dataset_auto_disable_log_dataset_idx", "dataset_id"),
+        sa.Index("dataset_auto_disable_log_created_atx", "created_at"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     dataset_id = mapped_column(StringUUID, nullable=False)
     document_id = mapped_column(StringUUID, nullable=False)
-    notified: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    notified: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
     created_at: Mapped[datetime] = mapped_column(
-        DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
 
 
 class RateLimitLog(Base):
     __tablename__ = "rate_limit_logs"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="rate_limit_log_pkey"),
-        db.Index("rate_limit_log_tenant_idx", "tenant_id"),
-        db.Index("rate_limit_log_operation_idx", "operation"),
+        sa.PrimaryKeyConstraint("id", name="rate_limit_log_pkey"),
+        sa.Index("rate_limit_log_tenant_idx", "tenant_id"),
+        sa.Index("rate_limit_log_operation_idx", "operation"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     subscription_plan: Mapped[str] = mapped_column(String(255), nullable=False)
     operation: Mapped[str] = mapped_column(String(255), nullable=False)
     created_at: Mapped[datetime] = mapped_column(
-        DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
 
 
 class DatasetMetadata(Base):
     __tablename__ = "dataset_metadatas"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_metadata_pkey"),
-        db.Index("dataset_metadata_tenant_idx", "tenant_id"),
-        db.Index("dataset_metadata_dataset_idx", "dataset_id"),
+        sa.PrimaryKeyConstraint("id", name="dataset_metadata_pkey"),
+        sa.Index("dataset_metadata_tenant_idx", "tenant_id"),
+        sa.Index("dataset_metadata_dataset_idx", "dataset_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     dataset_id = mapped_column(StringUUID, nullable=False)
     type: Mapped[str] = mapped_column(String(255), nullable=False)
     name: Mapped[str] = mapped_column(String(255), nullable=False)
     created_at: Mapped[datetime] = mapped_column(
-        DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
     updated_at: Mapped[datetime] = mapped_column(
-        DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
     created_by = mapped_column(StringUUID, nullable=False)
     updated_by = mapped_column(StringUUID, nullable=True)
@@ -1141,14 +1142,14 @@ class DatasetMetadata(Base):
 class DatasetMetadataBinding(Base):
     __tablename__ = "dataset_metadata_bindings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_metadata_binding_pkey"),
-        db.Index("dataset_metadata_binding_tenant_idx", "tenant_id"),
-        db.Index("dataset_metadata_binding_dataset_idx", "dataset_id"),
-        db.Index("dataset_metadata_binding_metadata_idx", "metadata_id"),
-        db.Index("dataset_metadata_binding_document_idx", "document_id"),
+        sa.PrimaryKeyConstraint("id", name="dataset_metadata_binding_pkey"),
+        sa.Index("dataset_metadata_binding_tenant_idx", "tenant_id"),
+        sa.Index("dataset_metadata_binding_dataset_idx", "dataset_id"),
+        sa.Index("dataset_metadata_binding_metadata_idx", "metadata_id"),
+        sa.Index("dataset_metadata_binding_document_idx", "document_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     dataset_id = mapped_column(StringUUID, nullable=False)
     metadata_id = mapped_column(StringUUID, nullable=False)

api/models/model.py

@@ -35,10 +35,10 @@ from .types import StringUUID
 
 class DifySetup(Base):
     __tablename__ = "dify_setups"
-    __table_args__ = (db.PrimaryKeyConstraint("version", name="dify_setup_pkey"),)
+    __table_args__ = (sa.PrimaryKeyConstraint("version", name="dify_setup_pkey"),)
 
     version: Mapped[str] = mapped_column(String(255), nullable=False)
-    setup_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    setup_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
 
 class AppMode(StrEnum):
@@ -69,33 +69,33 @@ class IconType(Enum):
 
 class App(Base):
     __tablename__ = "apps"
-    __table_args__ = (db.PrimaryKeyConstraint("id", name="app_pkey"), db.Index("app_tenant_id_idx", "tenant_id"))
+    __table_args__ = (sa.PrimaryKeyConstraint("id", name="app_pkey"), sa.Index("app_tenant_id_idx", "tenant_id"))
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID)
     name: Mapped[str] = mapped_column(String(255))
-    description: Mapped[str] = mapped_column(db.Text, server_default=db.text("''::character varying"))
+    description: Mapped[str] = mapped_column(sa.Text, server_default=sa.text("''::character varying"))
     mode: Mapped[str] = mapped_column(String(255))
     icon_type: Mapped[Optional[str]] = mapped_column(String(255))  # image, emoji
     icon = db.Column(String(255))
     icon_background: Mapped[Optional[str]] = mapped_column(String(255))
     app_model_config_id = mapped_column(StringUUID, nullable=True)
     workflow_id = mapped_column(StringUUID, nullable=True)
-    status: Mapped[str] = mapped_column(String(255), server_default=db.text("'normal'::character varying"))
-    enable_site: Mapped[bool] = mapped_column(db.Boolean)
-    enable_api: Mapped[bool] = mapped_column(db.Boolean)
-    api_rpm: Mapped[int] = mapped_column(db.Integer, server_default=db.text("0"))
-    api_rph: Mapped[int] = mapped_column(db.Integer, server_default=db.text("0"))
-    is_demo: Mapped[bool] = mapped_column(db.Boolean, server_default=db.text("false"))
-    is_public: Mapped[bool] = mapped_column(db.Boolean, server_default=db.text("false"))
-    is_universal: Mapped[bool] = mapped_column(db.Boolean, server_default=db.text("false"))
-    tracing = mapped_column(db.Text, nullable=True)
+    status: Mapped[str] = mapped_column(String(255), server_default=sa.text("'normal'::character varying"))
+    enable_site: Mapped[bool] = mapped_column(sa.Boolean)
+    enable_api: Mapped[bool] = mapped_column(sa.Boolean)
+    api_rpm: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"))
+    api_rph: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"))
+    is_demo: Mapped[bool] = mapped_column(sa.Boolean, server_default=sa.text("false"))
+    is_public: Mapped[bool] = mapped_column(sa.Boolean, server_default=sa.text("false"))
+    is_universal: Mapped[bool] = mapped_column(sa.Boolean, server_default=sa.text("false"))
+    tracing = mapped_column(sa.Text, nullable=True)
     max_active_requests: Mapped[Optional[int]]
     created_by = mapped_column(StringUUID, nullable=True)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
     updated_by = mapped_column(StringUUID, nullable=True)
-    updated_at: Mapped[datetime] = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    use_icon_as_answer_icon: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    updated_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    use_icon_as_answer_icon: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
 
     @property
     def desc_or_prompt(self):
@@ -302,36 +302,36 @@ class App(Base):
 
 class AppModelConfig(Base):
     __tablename__ = "app_model_configs"
-    __table_args__ = (db.PrimaryKeyConstraint("id", name="app_model_config_pkey"), db.Index("app_app_id_idx", "app_id"))
+    __table_args__ = (sa.PrimaryKeyConstraint("id", name="app_model_config_pkey"), sa.Index("app_app_id_idx", "app_id"))
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     provider = mapped_column(String(255), nullable=True)
     model_id = mapped_column(String(255), nullable=True)
-    configs = mapped_column(db.JSON, nullable=True)
+    configs = mapped_column(sa.JSON, nullable=True)
     created_by = mapped_column(StringUUID, nullable=True)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
     updated_by = mapped_column(StringUUID, nullable=True)
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    opening_statement = mapped_column(db.Text)
-    suggested_questions = mapped_column(db.Text)
-    suggested_questions_after_answer = mapped_column(db.Text)
-    speech_to_text = mapped_column(db.Text)
-    text_to_speech = mapped_column(db.Text)
-    more_like_this = mapped_column(db.Text)
-    model = mapped_column(db.Text)
-    user_input_form = mapped_column(db.Text)
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    opening_statement = mapped_column(sa.Text)
+    suggested_questions = mapped_column(sa.Text)
+    suggested_questions_after_answer = mapped_column(sa.Text)
+    speech_to_text = mapped_column(sa.Text)
+    text_to_speech = mapped_column(sa.Text)
+    more_like_this = mapped_column(sa.Text)
+    model = mapped_column(sa.Text)
+    user_input_form = mapped_column(sa.Text)
     dataset_query_variable = mapped_column(String(255))
-    pre_prompt = mapped_column(db.Text)
-    agent_mode = mapped_column(db.Text)
-    sensitive_word_avoidance = mapped_column(db.Text)
-    retriever_resource = mapped_column(db.Text)
-    prompt_type = mapped_column(String(255), nullable=False, server_default=db.text("'simple'::character varying"))
-    chat_prompt_config = mapped_column(db.Text)
-    completion_prompt_config = mapped_column(db.Text)
-    dataset_configs = mapped_column(db.Text)
-    external_data_tools = mapped_column(db.Text)
-    file_upload = mapped_column(db.Text)
+    pre_prompt = mapped_column(sa.Text)
+    agent_mode = mapped_column(sa.Text)
+    sensitive_word_avoidance = mapped_column(sa.Text)
+    retriever_resource = mapped_column(sa.Text)
+    prompt_type = mapped_column(String(255), nullable=False, server_default=sa.text("'simple'::character varying"))
+    chat_prompt_config = mapped_column(sa.Text)
+    completion_prompt_config = mapped_column(sa.Text)
+    dataset_configs = mapped_column(sa.Text)
+    external_data_tools = mapped_column(sa.Text)
+    file_upload = mapped_column(sa.Text)
 
     @property
     def app(self):
@@ -553,24 +553,24 @@ class AppModelConfig(Base):
 class RecommendedApp(Base):
     __tablename__ = "recommended_apps"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="recommended_app_pkey"),
-        db.Index("recommended_app_app_id_idx", "app_id"),
-        db.Index("recommended_app_is_listed_idx", "is_listed", "language"),
+        sa.PrimaryKeyConstraint("id", name="recommended_app_pkey"),
+        sa.Index("recommended_app_app_id_idx", "app_id"),
+        sa.Index("recommended_app_is_listed_idx", "is_listed", "language"),
     )
 
-    id = mapped_column(StringUUID, primary_key=True, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
-    description = mapped_column(db.JSON, nullable=False)
+    description = mapped_column(sa.JSON, nullable=False)
     copyright: Mapped[str] = mapped_column(String(255), nullable=False)
     privacy_policy: Mapped[str] = mapped_column(String(255), nullable=False)
     custom_disclaimer: Mapped[str] = mapped_column(sa.TEXT, default="")
     category: Mapped[str] = mapped_column(String(255), nullable=False)
-    position: Mapped[int] = mapped_column(db.Integer, nullable=False, default=0)
-    is_listed: Mapped[bool] = mapped_column(db.Boolean, nullable=False, default=True)
-    install_count: Mapped[int] = mapped_column(db.Integer, nullable=False, default=0)
-    language = mapped_column(String(255), nullable=False, server_default=db.text("'en-US'::character varying"))
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    position: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)
+    is_listed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=True)
+    install_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)
+    language = mapped_column(String(255), nullable=False, server_default=sa.text("'en-US'::character varying"))
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
     @property
     def app(self):
@@ -581,20 +581,20 @@ class RecommendedApp(Base):
 class InstalledApp(Base):
     __tablename__ = "installed_apps"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="installed_app_pkey"),
-        db.Index("installed_app_tenant_id_idx", "tenant_id"),
-        db.Index("installed_app_app_id_idx", "app_id"),
-        db.UniqueConstraint("tenant_id", "app_id", name="unique_tenant_app"),
+        sa.PrimaryKeyConstraint("id", name="installed_app_pkey"),
+        sa.Index("installed_app_tenant_id_idx", "tenant_id"),
+        sa.Index("installed_app_app_id_idx", "app_id"),
+        sa.UniqueConstraint("tenant_id", "app_id", name="unique_tenant_app"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     app_id = mapped_column(StringUUID, nullable=False)
     app_owner_tenant_id = mapped_column(StringUUID, nullable=False)
-    position: Mapped[int] = mapped_column(db.Integer, nullable=False, default=0)
-    is_pinned: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
-    last_used_at = mapped_column(db.DateTime, nullable=True)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    position: Mapped[int] = mapped_column(sa.Integer, nullable=False, default=0)
+    is_pinned: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
+    last_used_at = mapped_column(sa.DateTime, nullable=True)
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
     @property
     def app(self):
@@ -610,23 +610,23 @@ class InstalledApp(Base):
 class Conversation(Base):
     __tablename__ = "conversations"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="conversation_pkey"),
-        db.Index("conversation_app_from_user_idx", "app_id", "from_source", "from_end_user_id"),
+        sa.PrimaryKeyConstraint("id", name="conversation_pkey"),
+        sa.Index("conversation_app_from_user_idx", "app_id", "from_source", "from_end_user_id"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     app_model_config_id = mapped_column(StringUUID, nullable=True)
     model_provider = mapped_column(String(255), nullable=True)
-    override_model_configs = mapped_column(db.Text)
+    override_model_configs = mapped_column(sa.Text)
     model_id = mapped_column(String(255), nullable=True)
     mode: Mapped[str] = mapped_column(String(255))
     name: Mapped[str] = mapped_column(String(255), nullable=False)
-    summary = mapped_column(db.Text)
-    _inputs: Mapped[dict] = mapped_column("inputs", db.JSON)
-    introduction = mapped_column(db.Text)
-    system_instruction = mapped_column(db.Text)
-    system_instruction_tokens: Mapped[int] = mapped_column(db.Integer, nullable=False, server_default=db.text("0"))
+    summary = mapped_column(sa.Text)
+    _inputs: Mapped[dict] = mapped_column("inputs", sa.JSON)
+    introduction = mapped_column(sa.Text)
+    system_instruction = mapped_column(sa.Text)
+    system_instruction_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"))
     status: Mapped[str] = mapped_column(String(255), nullable=False)
 
     # The `invoke_from` records how the conversation is created.
@@ -639,18 +639,18 @@ class Conversation(Base):
     from_source: Mapped[str] = mapped_column(String(255), nullable=False)
     from_end_user_id = mapped_column(StringUUID)
     from_account_id = mapped_column(StringUUID)
-    read_at = mapped_column(db.DateTime)
+    read_at = mapped_column(sa.DateTime)
     read_account_id = mapped_column(StringUUID)
     dialogue_count: Mapped[int] = mapped_column(default=0)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
     messages = db.relationship("Message", backref="conversation", lazy="select", passive_deletes="all")
     message_annotations = db.relationship(
         "MessageAnnotation", backref="conversation", lazy="select", passive_deletes="all"
     )
 
-    is_deleted: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    is_deleted: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
 
     @property
     def inputs(self):
@@ -892,36 +892,36 @@ class Message(Base):
         Index("message_created_at_idx", "created_at"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     model_provider = mapped_column(String(255), nullable=True)
     model_id = mapped_column(String(255), nullable=True)
-    override_model_configs = mapped_column(db.Text)
-    conversation_id = mapped_column(StringUUID, db.ForeignKey("conversations.id"), nullable=False)
-    _inputs: Mapped[dict] = mapped_column("inputs", db.JSON)
-    query: Mapped[str] = mapped_column(db.Text, nullable=False)
-    message = mapped_column(db.JSON, nullable=False)
-    message_tokens: Mapped[int] = mapped_column(db.Integer, nullable=False, server_default=db.text("0"))
-    message_unit_price = mapped_column(db.Numeric(10, 4), nullable=False)
-    message_price_unit = mapped_column(db.Numeric(10, 7), nullable=False, server_default=db.text("0.001"))
-    answer: Mapped[str] = db.Column(db.Text, nullable=False)  # TODO make it mapped_column
-    answer_tokens: Mapped[int] = mapped_column(db.Integer, nullable=False, server_default=db.text("0"))
-    answer_unit_price = mapped_column(db.Numeric(10, 4), nullable=False)
-    answer_price_unit = mapped_column(db.Numeric(10, 7), nullable=False, server_default=db.text("0.001"))
+    override_model_configs = mapped_column(sa.Text)
+    conversation_id = mapped_column(StringUUID, sa.ForeignKey("conversations.id"), nullable=False)
+    _inputs: Mapped[dict] = mapped_column("inputs", sa.JSON)
+    query: Mapped[str] = mapped_column(sa.Text, nullable=False)
+    message = mapped_column(sa.JSON, nullable=False)
+    message_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"))
+    message_unit_price = mapped_column(sa.Numeric(10, 4), nullable=False)
+    message_price_unit = mapped_column(sa.Numeric(10, 7), nullable=False, server_default=sa.text("0.001"))
+    answer: Mapped[str] = db.Column(sa.Text, nullable=False)  # TODO make it mapped_column
+    answer_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"))
+    answer_unit_price = mapped_column(sa.Numeric(10, 4), nullable=False)
+    answer_price_unit = mapped_column(sa.Numeric(10, 7), nullable=False, server_default=sa.text("0.001"))
     parent_message_id = mapped_column(StringUUID, nullable=True)
-    provider_response_latency = mapped_column(db.Float, nullable=False, server_default=db.text("0"))
-    total_price = mapped_column(db.Numeric(10, 7))
+    provider_response_latency = mapped_column(sa.Float, nullable=False, server_default=sa.text("0"))
+    total_price = mapped_column(sa.Numeric(10, 7))
     currency: Mapped[str] = mapped_column(String(255), nullable=False)
-    status = mapped_column(String(255), nullable=False, server_default=db.text("'normal'::character varying"))
-    error = mapped_column(db.Text)
-    message_metadata = mapped_column(db.Text)
+    status = mapped_column(String(255), nullable=False, server_default=sa.text("'normal'::character varying"))
+    error = mapped_column(sa.Text)
+    message_metadata = mapped_column(sa.Text)
     invoke_from: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
     from_source: Mapped[str] = mapped_column(String(255), nullable=False)
     from_end_user_id: Mapped[Optional[str]] = mapped_column(StringUUID)
     from_account_id: Mapped[Optional[str]] = mapped_column(StringUUID)
-    created_at: Mapped[datetime] = mapped_column(db.DateTime, server_default=func.current_timestamp())
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    agent_based: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    created_at: Mapped[datetime] = mapped_column(sa.DateTime, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    agent_based: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
     workflow_run_id: Mapped[Optional[str]] = mapped_column(StringUUID)
 
     @property
@@ -1228,23 +1228,23 @@ class Message(Base):
 class MessageFeedback(Base):
     __tablename__ = "message_feedbacks"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="message_feedback_pkey"),
-        db.Index("message_feedback_app_idx", "app_id"),
-        db.Index("message_feedback_message_idx", "message_id", "from_source"),
-        db.Index("message_feedback_conversation_idx", "conversation_id", "from_source", "rating"),
+        sa.PrimaryKeyConstraint("id", name="message_feedback_pkey"),
+        sa.Index("message_feedback_app_idx", "app_id"),
+        sa.Index("message_feedback_message_idx", "message_id", "from_source"),
+        sa.Index("message_feedback_conversation_idx", "conversation_id", "from_source", "rating"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     conversation_id = mapped_column(StringUUID, nullable=False)
     message_id = mapped_column(StringUUID, nullable=False)
     rating: Mapped[str] = mapped_column(String(255), nullable=False)
-    content = mapped_column(db.Text)
+    content = mapped_column(sa.Text)
     from_source: Mapped[str] = mapped_column(String(255), nullable=False)
     from_end_user_id = mapped_column(StringUUID)
     from_account_id = mapped_column(StringUUID)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
     @property
     def from_account(self):
@@ -1270,9 +1270,9 @@ class MessageFeedback(Base):
 class MessageFile(Base):
     __tablename__ = "message_files"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="message_file_pkey"),
-        db.Index("message_file_message_idx", "message_id"),
-        db.Index("message_file_created_by_idx", "created_by"),
+        sa.PrimaryKeyConstraint("id", name="message_file_pkey"),
+        sa.Index("message_file_message_idx", "message_id"),
+        sa.Index("message_file_created_by_idx", "created_by"),
     )
 
     def __init__(
@@ -1296,37 +1296,37 @@ class MessageFile(Base):
         self.created_by_role = created_by_role.value
         self.created_by = created_by
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     message_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     type: Mapped[str] = mapped_column(String(255), nullable=False)
     transfer_method: Mapped[str] = mapped_column(String(255), nullable=False)
-    url: Mapped[Optional[str]] = mapped_column(db.Text, nullable=True)
+    url: Mapped[Optional[str]] = mapped_column(sa.Text, nullable=True)
     belongs_to: Mapped[Optional[str]] = mapped_column(String(255), nullable=True)
     upload_file_id: Mapped[Optional[str]] = mapped_column(StringUUID, nullable=True)
     created_by_role: Mapped[str] = mapped_column(String(255), nullable=False)
     created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
-    created_at: Mapped[datetime] = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
 
 class MessageAnnotation(Base):
     __tablename__ = "message_annotations"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="message_annotation_pkey"),
-        db.Index("message_annotation_app_idx", "app_id"),
-        db.Index("message_annotation_conversation_idx", "conversation_id"),
-        db.Index("message_annotation_message_idx", "message_id"),
+        sa.PrimaryKeyConstraint("id", name="message_annotation_pkey"),
+        sa.Index("message_annotation_app_idx", "app_id"),
+        sa.Index("message_annotation_conversation_idx", "conversation_id"),
+        sa.Index("message_annotation_message_idx", "message_id"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id: Mapped[str] = mapped_column(StringUUID)
-    conversation_id: Mapped[Optional[str]] = mapped_column(StringUUID, db.ForeignKey("conversations.id"))
+    conversation_id: Mapped[Optional[str]] = mapped_column(StringUUID, sa.ForeignKey("conversations.id"))
     message_id: Mapped[Optional[str]] = mapped_column(StringUUID)
-    question = db.Column(db.Text, nullable=True)
-    content = mapped_column(db.Text, nullable=False)
-    hit_count: Mapped[int] = mapped_column(db.Integer, nullable=False, server_default=db.text("0"))
+    question = db.Column(sa.Text, nullable=True)
+    content = mapped_column(sa.Text, nullable=False)
+    hit_count: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"))
     account_id = mapped_column(StringUUID, nullable=False)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
     @property
     def account(self):
@@ -1342,24 +1342,24 @@ class MessageAnnotation(Base):
 class AppAnnotationHitHistory(Base):
     __tablename__ = "app_annotation_hit_histories"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="app_annotation_hit_histories_pkey"),
-        db.Index("app_annotation_hit_histories_app_idx", "app_id"),
-        db.Index("app_annotation_hit_histories_account_idx", "account_id"),
-        db.Index("app_annotation_hit_histories_annotation_idx", "annotation_id"),
-        db.Index("app_annotation_hit_histories_message_idx", "message_id"),
+        sa.PrimaryKeyConstraint("id", name="app_annotation_hit_histories_pkey"),
+        sa.Index("app_annotation_hit_histories_app_idx", "app_id"),
+        sa.Index("app_annotation_hit_histories_account_idx", "account_id"),
+        sa.Index("app_annotation_hit_histories_annotation_idx", "annotation_id"),
+        sa.Index("app_annotation_hit_histories_message_idx", "message_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     annotation_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
-    source = mapped_column(db.Text, nullable=False)
-    question = mapped_column(db.Text, nullable=False)
+    source = mapped_column(sa.Text, nullable=False)
+    question = mapped_column(sa.Text, nullable=False)
     account_id = mapped_column(StringUUID, nullable=False)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    score = mapped_column(Float, nullable=False, server_default=db.text("0"))
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    score = mapped_column(Float, nullable=False, server_default=sa.text("0"))
     message_id = mapped_column(StringUUID, nullable=False)
-    annotation_question = mapped_column(db.Text, nullable=False)
-    annotation_content = mapped_column(db.Text, nullable=False)
+    annotation_question = mapped_column(sa.Text, nullable=False)
+    annotation_content = mapped_column(sa.Text, nullable=False)
 
     @property
     def account(self):
@@ -1380,18 +1380,18 @@ class AppAnnotationHitHistory(Base):
 class AppAnnotationSetting(Base):
     __tablename__ = "app_annotation_settings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="app_annotation_settings_pkey"),
-        db.Index("app_annotation_settings_app_idx", "app_id"),
+        sa.PrimaryKeyConstraint("id", name="app_annotation_settings_pkey"),
+        sa.Index("app_annotation_settings_app_idx", "app_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
-    score_threshold = mapped_column(Float, nullable=False, server_default=db.text("0"))
+    score_threshold = mapped_column(Float, nullable=False, server_default=sa.text("0"))
     collection_binding_id = mapped_column(StringUUID, nullable=False)
     created_user_id = mapped_column(StringUUID, nullable=False)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
     updated_user_id = mapped_column(StringUUID, nullable=False)
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
     @property
     def collection_binding_detail(self):
@@ -1408,58 +1408,58 @@ class AppAnnotationSetting(Base):
 class OperationLog(Base):
     __tablename__ = "operation_logs"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="operation_log_pkey"),
-        db.Index("operation_log_account_action_idx", "tenant_id", "account_id", "action"),
+        sa.PrimaryKeyConstraint("id", name="operation_log_pkey"),
+        sa.Index("operation_log_account_action_idx", "tenant_id", "account_id", "action"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     account_id = mapped_column(StringUUID, nullable=False)
     action: Mapped[str] = mapped_column(String(255), nullable=False)
-    content = mapped_column(db.JSON)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    content = mapped_column(sa.JSON)
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
     created_ip: Mapped[str] = mapped_column(String(255), nullable=False)
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
 
 class EndUser(Base, UserMixin):
     __tablename__ = "end_users"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="end_user_pkey"),
-        db.Index("end_user_session_id_idx", "session_id", "type"),
-        db.Index("end_user_tenant_session_id_idx", "tenant_id", "session_id", "type"),
+        sa.PrimaryKeyConstraint("id", name="end_user_pkey"),
+        sa.Index("end_user_session_id_idx", "session_id", "type"),
+        sa.Index("end_user_tenant_session_id_idx", "tenant_id", "session_id", "type"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     app_id = mapped_column(StringUUID, nullable=True)
     type: Mapped[str] = mapped_column(String(255), nullable=False)
     external_user_id = mapped_column(String(255), nullable=True)
     name = mapped_column(String(255))
-    is_anonymous: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("true"))
+    is_anonymous: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
     session_id: Mapped[str] = mapped_column()
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
 
 class AppMCPServer(Base):
     __tablename__ = "app_mcp_servers"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="app_mcp_server_pkey"),
-        db.UniqueConstraint("tenant_id", "app_id", name="unique_app_mcp_server_tenant_app_id"),
-        db.UniqueConstraint("server_code", name="unique_app_mcp_server_server_code"),
+        sa.PrimaryKeyConstraint("id", name="app_mcp_server_pkey"),
+        sa.UniqueConstraint("tenant_id", "app_id", name="unique_app_mcp_server_tenant_app_id"),
+        sa.UniqueConstraint("server_code", name="unique_app_mcp_server_server_code"),
     )
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     app_id = mapped_column(StringUUID, nullable=False)
     name: Mapped[str] = mapped_column(String(255), nullable=False)
     description: Mapped[str] = mapped_column(String(255), nullable=False)
     server_code: Mapped[str] = mapped_column(String(255), nullable=False)
-    status = mapped_column(String(255), nullable=False, server_default=db.text("'normal'::character varying"))
-    parameters = mapped_column(db.Text, nullable=False)
+    status = mapped_column(String(255), nullable=False, server_default=sa.text("'normal'::character varying"))
+    parameters = mapped_column(sa.Text, nullable=False)
 
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
     @staticmethod
     def generate_server_code(n):
@@ -1478,34 +1478,34 @@ class AppMCPServer(Base):
 class Site(Base):
     __tablename__ = "sites"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="site_pkey"),
-        db.Index("site_app_id_idx", "app_id"),
-        db.Index("site_code_idx", "code", "status"),
+        sa.PrimaryKeyConstraint("id", name="site_pkey"),
+        sa.Index("site_app_id_idx", "app_id"),
+        sa.Index("site_code_idx", "code", "status"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     title: Mapped[str] = mapped_column(String(255), nullable=False)
     icon_type = mapped_column(String(255), nullable=True)
     icon = mapped_column(String(255))
     icon_background = mapped_column(String(255))
-    description = mapped_column(db.Text)
+    description = mapped_column(sa.Text)
     default_language: Mapped[str] = mapped_column(String(255), nullable=False)
     chat_color_theme = mapped_column(String(255))
-    chat_color_theme_inverted: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    chat_color_theme_inverted: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
     copyright = mapped_column(String(255))
     privacy_policy = mapped_column(String(255))
-    show_workflow_steps: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("true"))
-    use_icon_as_answer_icon: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    show_workflow_steps: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
+    use_icon_as_answer_icon: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
     _custom_disclaimer: Mapped[str] = mapped_column("custom_disclaimer", sa.TEXT, default="")
     customize_domain = mapped_column(String(255))
     customize_token_strategy: Mapped[str] = mapped_column(String(255), nullable=False)
-    prompt_public: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
-    status = mapped_column(String(255), nullable=False, server_default=db.text("'normal'::character varying"))
+    prompt_public: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
+    status = mapped_column(String(255), nullable=False, server_default=sa.text("'normal'::character varying"))
     created_by = mapped_column(StringUUID, nullable=True)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
     updated_by = mapped_column(StringUUID, nullable=True)
-    updated_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
     code = mapped_column(String(255))
 
     @property
@@ -1535,19 +1535,19 @@ class Site(Base):
 class ApiToken(Base):
     __tablename__ = "api_tokens"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="api_token_pkey"),
-        db.Index("api_token_app_id_type_idx", "app_id", "type"),
-        db.Index("api_token_token_idx", "token", "type"),
-        db.Index("api_token_tenant_idx", "tenant_id", "type"),
+        sa.PrimaryKeyConstraint("id", name="api_token_pkey"),
+        sa.Index("api_token_app_id_type_idx", "app_id", "type"),
+        sa.Index("api_token_token_idx", "token", "type"),
+        sa.Index("api_token_tenant_idx", "tenant_id", "type"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=True)
     tenant_id = mapped_column(StringUUID, nullable=True)
     type = mapped_column(String(16), nullable=False)
     token: Mapped[str] = mapped_column(String(255), nullable=False)
-    last_used_at = mapped_column(db.DateTime, nullable=True)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    last_used_at = mapped_column(sa.DateTime, nullable=True)
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
     @staticmethod
     def generate_api_key(prefix, n):
@@ -1561,26 +1561,26 @@ class ApiToken(Base):
 class UploadFile(Base):
     __tablename__ = "upload_files"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="upload_file_pkey"),
-        db.Index("upload_file_tenant_idx", "tenant_id"),
+        sa.PrimaryKeyConstraint("id", name="upload_file_pkey"),
+        sa.Index("upload_file_tenant_idx", "tenant_id"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     storage_type: Mapped[str] = mapped_column(String(255), nullable=False)
     key: Mapped[str] = mapped_column(String(255), nullable=False)
     name: Mapped[str] = mapped_column(String(255), nullable=False)
-    size: Mapped[int] = mapped_column(db.Integer, nullable=False)
+    size: Mapped[int] = mapped_column(sa.Integer, nullable=False)
     extension: Mapped[str] = mapped_column(String(255), nullable=False)
     mime_type: Mapped[str] = mapped_column(String(255), nullable=True)
     created_by_role: Mapped[str] = mapped_column(
-        String(255), nullable=False, server_default=db.text("'account'::character varying")
+        String(255), nullable=False, server_default=sa.text("'account'::character varying")
     )
     created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
-    created_at: Mapped[datetime] = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
-    used: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    created_at: Mapped[datetime] = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
+    used: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
     used_by: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
-    used_at: Mapped[datetime | None] = mapped_column(db.DateTime, nullable=True)
+    used_at: Mapped[datetime | None] = mapped_column(sa.DateTime, nullable=True)
     hash: Mapped[str | None] = mapped_column(String(255), nullable=True)
     source_url: Mapped[str] = mapped_column(sa.TEXT, default="")
 
@@ -1623,71 +1623,71 @@ class UploadFile(Base):
 class ApiRequest(Base):
     __tablename__ = "api_requests"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="api_request_pkey"),
-        db.Index("api_request_token_idx", "tenant_id", "api_token_id"),
+        sa.PrimaryKeyConstraint("id", name="api_request_pkey"),
+        sa.Index("api_request_token_idx", "tenant_id", "api_token_id"),
     )
 
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     api_token_id = mapped_column(StringUUID, nullable=False)
     path: Mapped[str] = mapped_column(String(255), nullable=False)
-    request = mapped_column(db.Text, nullable=True)
-    response = mapped_column(db.Text, nullable=True)
+    request = mapped_column(sa.Text, nullable=True)
+    response = mapped_column(sa.Text, nullable=True)
     ip: Mapped[str] = mapped_column(String(255), nullable=False)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
 
 class MessageChain(Base):
     __tablename__ = "message_chains"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="message_chain_pkey"),
-        db.Index("message_chain_message_id_idx", "message_id"),
+        sa.PrimaryKeyConstraint("id", name="message_chain_pkey"),
+        sa.Index("message_chain_message_id_idx", "message_id"),
     )
 
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     message_id = mapped_column(StringUUID, nullable=False)
     type: Mapped[str] = mapped_column(String(255), nullable=False)
-    input = mapped_column(db.Text, nullable=True)
-    output = mapped_column(db.Text, nullable=True)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())
+    input = mapped_column(sa.Text, nullable=True)
+    output = mapped_column(sa.Text, nullable=True)
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=db.func.current_timestamp())
 
 
 class MessageAgentThought(Base):
     __tablename__ = "message_agent_thoughts"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="message_agent_thought_pkey"),
-        db.Index("message_agent_thought_message_id_idx", "message_id"),
-        db.Index("message_agent_thought_message_chain_id_idx", "message_chain_id"),
+        sa.PrimaryKeyConstraint("id", name="message_agent_thought_pkey"),
+        sa.Index("message_agent_thought_message_id_idx", "message_id"),
+        sa.Index("message_agent_thought_message_chain_id_idx", "message_chain_id"),
     )
 
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     message_id = mapped_column(StringUUID, nullable=False)
     message_chain_id = mapped_column(StringUUID, nullable=True)
-    position: Mapped[int] = mapped_column(db.Integer, nullable=False)
-    thought = mapped_column(db.Text, nullable=True)
-    tool = mapped_column(db.Text, nullable=True)
-    tool_labels_str = mapped_column(db.Text, nullable=False, server_default=db.text("'{}'::text"))
-    tool_meta_str = mapped_column(db.Text, nullable=False, server_default=db.text("'{}'::text"))
-    tool_input = mapped_column(db.Text, nullable=True)
-    observation = mapped_column(db.Text, nullable=True)
+    position: Mapped[int] = mapped_column(sa.Integer, nullable=False)
+    thought = mapped_column(sa.Text, nullable=True)
+    tool = mapped_column(sa.Text, nullable=True)
+    tool_labels_str = mapped_column(sa.Text, nullable=False, server_default=sa.text("'{}'::text"))
+    tool_meta_str = mapped_column(sa.Text, nullable=False, server_default=sa.text("'{}'::text"))
+    tool_input = mapped_column(sa.Text, nullable=True)
+    observation = mapped_column(sa.Text, nullable=True)
     # plugin_id = mapped_column(StringUUID, nullable=True)  ## for future design
-    tool_process_data = mapped_column(db.Text, nullable=True)
-    message = mapped_column(db.Text, nullable=True)
-    message_token: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)
-    message_unit_price = mapped_column(db.Numeric, nullable=True)
-    message_price_unit = mapped_column(db.Numeric(10, 7), nullable=False, server_default=db.text("0.001"))
-    message_files = mapped_column(db.Text, nullable=True)
-    answer = db.Column(db.Text, nullable=True)
-    answer_token: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)
-    answer_unit_price = mapped_column(db.Numeric, nullable=True)
-    answer_price_unit = mapped_column(db.Numeric(10, 7), nullable=False, server_default=db.text("0.001"))
-    tokens: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)
-    total_price = mapped_column(db.Numeric, nullable=True)
+    tool_process_data = mapped_column(sa.Text, nullable=True)
+    message = mapped_column(sa.Text, nullable=True)
+    message_token: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
+    message_unit_price = mapped_column(sa.Numeric, nullable=True)
+    message_price_unit = mapped_column(sa.Numeric(10, 7), nullable=False, server_default=sa.text("0.001"))
+    message_files = mapped_column(sa.Text, nullable=True)
+    answer = db.Column(sa.Text, nullable=True)
+    answer_token: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
+    answer_unit_price = mapped_column(sa.Numeric, nullable=True)
+    answer_price_unit = mapped_column(sa.Numeric(10, 7), nullable=False, server_default=sa.text("0.001"))
+    tokens: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
+    total_price = mapped_column(sa.Numeric, nullable=True)
     currency = mapped_column(String, nullable=True)
-    latency: Mapped[Optional[float]] = mapped_column(db.Float, nullable=True)
+    latency: Mapped[Optional[float]] = mapped_column(sa.Float, nullable=True)
     created_by_role = mapped_column(String, nullable=False)
     created_by = mapped_column(StringUUID, nullable=False)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=db.func.current_timestamp())
 
     @property
     def files(self) -> list:
@@ -1769,80 +1769,80 @@ class MessageAgentThought(Base):
 class DatasetRetrieverResource(Base):
     __tablename__ = "dataset_retriever_resources"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="dataset_retriever_resource_pkey"),
-        db.Index("dataset_retriever_resource_message_id_idx", "message_id"),
+        sa.PrimaryKeyConstraint("id", name="dataset_retriever_resource_pkey"),
+        sa.Index("dataset_retriever_resource_message_id_idx", "message_id"),
     )
 
-    id = mapped_column(StringUUID, nullable=False, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, nullable=False, server_default=sa.text("uuid_generate_v4()"))
     message_id = mapped_column(StringUUID, nullable=False)
-    position: Mapped[int] = mapped_column(db.Integer, nullable=False)
+    position: Mapped[int] = mapped_column(sa.Integer, nullable=False)
     dataset_id = mapped_column(StringUUID, nullable=False)
-    dataset_name = mapped_column(db.Text, nullable=False)
+    dataset_name = mapped_column(sa.Text, nullable=False)
     document_id = mapped_column(StringUUID, nullable=True)
-    document_name = mapped_column(db.Text, nullable=False)
-    data_source_type = mapped_column(db.Text, nullable=True)
+    document_name = mapped_column(sa.Text, nullable=False)
+    data_source_type = mapped_column(sa.Text, nullable=True)
     segment_id = mapped_column(StringUUID, nullable=True)
-    score: Mapped[Optional[float]] = mapped_column(db.Float, nullable=True)
-    content = mapped_column(db.Text, nullable=False)
-    hit_count: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)
-    word_count: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)
-    segment_position: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)
-    index_node_hash = mapped_column(db.Text, nullable=True)
-    retriever_from = mapped_column(db.Text, nullable=False)
+    score: Mapped[Optional[float]] = mapped_column(sa.Float, nullable=True)
+    content = mapped_column(sa.Text, nullable=False)
+    hit_count: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
+    word_count: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
+    segment_position: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
+    index_node_hash = mapped_column(sa.Text, nullable=True)
+    retriever_from = mapped_column(sa.Text, nullable=False)
     created_by = mapped_column(StringUUID, nullable=False)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=db.func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=db.func.current_timestamp())
 
 
 class Tag(Base):
     __tablename__ = "tags"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tag_pkey"),
-        db.Index("tag_type_idx", "type"),
-        db.Index("tag_name_idx", "name"),
+        sa.PrimaryKeyConstraint("id", name="tag_pkey"),
+        sa.Index("tag_type_idx", "type"),
+        sa.Index("tag_name_idx", "name"),
     )
 
     TAG_TYPE_LIST = ["knowledge", "app"]
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=True)
     type = mapped_column(String(16), nullable=False)
     name: Mapped[str] = mapped_column(String(255), nullable=False)
     created_by = mapped_column(StringUUID, nullable=False)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
 
 class TagBinding(Base):
     __tablename__ = "tag_bindings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tag_binding_pkey"),
-        db.Index("tag_bind_target_id_idx", "target_id"),
-        db.Index("tag_bind_tag_id_idx", "tag_id"),
+        sa.PrimaryKeyConstraint("id", name="tag_binding_pkey"),
+        sa.Index("tag_bind_target_id_idx", "target_id"),
+        sa.Index("tag_bind_tag_id_idx", "tag_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=True)
     tag_id = mapped_column(StringUUID, nullable=True)
     target_id = mapped_column(StringUUID, nullable=True)
     created_by = mapped_column(StringUUID, nullable=False)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
 
 
 class TraceAppConfig(Base):
     __tablename__ = "trace_app_config"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tracing_app_config_pkey"),
-        db.Index("trace_app_config_app_id_idx", "app_id"),
+        sa.PrimaryKeyConstraint("id", name="tracing_app_config_pkey"),
+        sa.Index("trace_app_config_app_id_idx", "app_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     tracing_provider = mapped_column(String(255), nullable=True)
-    tracing_config = mapped_column(db.JSON, nullable=True)
-    created_at = mapped_column(db.DateTime, nullable=False, server_default=func.current_timestamp())
+    tracing_config = mapped_column(sa.JSON, nullable=True)
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at = mapped_column(
-        db.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
+        sa.DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
     )
-    is_active: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("true"))
+    is_active: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
 
     @property
     def tracing_config_dict(self):

api/models/provider.py

@@ -2,11 +2,11 @@ from datetime import datetime
 from enum import Enum
 from typing import Optional
 
+import sqlalchemy as sa
 from sqlalchemy import DateTime, String, func, text
 from sqlalchemy.orm import Mapped, mapped_column
 
 from .base import Base
-from .engine import db
 from .types import StringUUID
 
 
@@ -47,9 +47,9 @@ class Provider(Base):
 
     __tablename__ = "providers"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="provider_pkey"),
-        db.Index("provider_tenant_id_provider_idx", "tenant_id", "provider_name"),
-        db.UniqueConstraint(
+        sa.PrimaryKeyConstraint("id", name="provider_pkey"),
+        sa.Index("provider_tenant_id_provider_idx", "tenant_id", "provider_name"),
+        sa.UniqueConstraint(
             "tenant_id", "provider_name", "provider_type", "quota_type", name="unique_provider_name_type_quota"
         ),
     )
@@ -60,15 +60,15 @@ class Provider(Base):
     provider_type: Mapped[str] = mapped_column(
         String(40), nullable=False, server_default=text("'custom'::character varying")
     )
-    encrypted_config: Mapped[Optional[str]] = mapped_column(db.Text, nullable=True)
-    is_valid: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=text("false"))
+    encrypted_config: Mapped[Optional[str]] = mapped_column(sa.Text, nullable=True)
+    is_valid: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"))
     last_used: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True)
 
     quota_type: Mapped[Optional[str]] = mapped_column(
         String(40), nullable=True, server_default=text("''::character varying")
     )
-    quota_limit: Mapped[Optional[int]] = mapped_column(db.BigInteger, nullable=True)
-    quota_used: Mapped[Optional[int]] = mapped_column(db.BigInteger, default=0)
+    quota_limit: Mapped[Optional[int]] = mapped_column(sa.BigInteger, nullable=True)
+    quota_used: Mapped[Optional[int]] = mapped_column(sa.BigInteger, default=0)
 
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
@@ -104,9 +104,9 @@ class ProviderModel(Base):
 
     __tablename__ = "provider_models"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="provider_model_pkey"),
-        db.Index("provider_model_tenant_id_provider_idx", "tenant_id", "provider_name"),
-        db.UniqueConstraint(
+        sa.PrimaryKeyConstraint("id", name="provider_model_pkey"),
+        sa.Index("provider_model_tenant_id_provider_idx", "tenant_id", "provider_name"),
+        sa.UniqueConstraint(
             "tenant_id", "provider_name", "model_name", "model_type", name="unique_provider_model_name"
         ),
     )
@@ -116,8 +116,8 @@ class ProviderModel(Base):
     provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
     model_name: Mapped[str] = mapped_column(String(255), nullable=False)
     model_type: Mapped[str] = mapped_column(String(40), nullable=False)
-    encrypted_config: Mapped[Optional[str]] = mapped_column(db.Text, nullable=True)
-    is_valid: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=text("false"))
+    encrypted_config: Mapped[Optional[str]] = mapped_column(sa.Text, nullable=True)
+    is_valid: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"))
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
 
@@ -125,8 +125,8 @@ class ProviderModel(Base):
 class TenantDefaultModel(Base):
     __tablename__ = "tenant_default_models"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tenant_default_model_pkey"),
-        db.Index("tenant_default_model_tenant_id_provider_type_idx", "tenant_id", "provider_name", "model_type"),
+        sa.PrimaryKeyConstraint("id", name="tenant_default_model_pkey"),
+        sa.Index("tenant_default_model_tenant_id_provider_type_idx", "tenant_id", "provider_name", "model_type"),
     )
 
     id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
@@ -141,8 +141,8 @@ class TenantDefaultModel(Base):
 class TenantPreferredModelProvider(Base):
     __tablename__ = "tenant_preferred_model_providers"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tenant_preferred_model_provider_pkey"),
-        db.Index("tenant_preferred_model_provider_tenant_provider_idx", "tenant_id", "provider_name"),
+        sa.PrimaryKeyConstraint("id", name="tenant_preferred_model_provider_pkey"),
+        sa.Index("tenant_preferred_model_provider_tenant_provider_idx", "tenant_id", "provider_name"),
     )
 
     id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
@@ -156,8 +156,8 @@ class TenantPreferredModelProvider(Base):
 class ProviderOrder(Base):
     __tablename__ = "provider_orders"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="provider_order_pkey"),
-        db.Index("provider_order_tenant_provider_idx", "tenant_id", "provider_name"),
+        sa.PrimaryKeyConstraint("id", name="provider_order_pkey"),
+        sa.Index("provider_order_tenant_provider_idx", "tenant_id", "provider_name"),
     )
 
     id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
@@ -167,9 +167,9 @@ class ProviderOrder(Base):
     payment_product_id: Mapped[str] = mapped_column(String(191), nullable=False)
     payment_id: Mapped[Optional[str]] = mapped_column(String(191))
     transaction_id: Mapped[Optional[str]] = mapped_column(String(191))
-    quantity: Mapped[int] = mapped_column(db.Integer, nullable=False, server_default=text("1"))
+    quantity: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=text("1"))
     currency: Mapped[Optional[str]] = mapped_column(String(40))
-    total_amount: Mapped[Optional[int]] = mapped_column(db.Integer)
+    total_amount: Mapped[Optional[int]] = mapped_column(sa.Integer)
     payment_status: Mapped[str] = mapped_column(
         String(40), nullable=False, server_default=text("'wait_pay'::character varying")
     )
@@ -187,8 +187,8 @@ class ProviderModelSetting(Base):
 
     __tablename__ = "provider_model_settings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="provider_model_setting_pkey"),
-        db.Index("provider_model_setting_tenant_provider_model_idx", "tenant_id", "provider_name", "model_type"),
+        sa.PrimaryKeyConstraint("id", name="provider_model_setting_pkey"),
+        sa.Index("provider_model_setting_tenant_provider_model_idx", "tenant_id", "provider_name", "model_type"),
     )
 
     id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
@@ -196,8 +196,8 @@ class ProviderModelSetting(Base):
     provider_name: Mapped[str] = mapped_column(String(255), nullable=False)
     model_name: Mapped[str] = mapped_column(String(255), nullable=False)
     model_type: Mapped[str] = mapped_column(String(40), nullable=False)
-    enabled: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=text("true"))
-    load_balancing_enabled: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=text("false"))
+    enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("true"))
+    load_balancing_enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("false"))
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
 
@@ -209,8 +209,8 @@ class LoadBalancingModelConfig(Base):
 
     __tablename__ = "load_balancing_model_configs"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="load_balancing_model_config_pkey"),
-        db.Index("load_balancing_model_config_tenant_provider_model_idx", "tenant_id", "provider_name", "model_type"),
+        sa.PrimaryKeyConstraint("id", name="load_balancing_model_config_pkey"),
+        sa.Index("load_balancing_model_config_tenant_provider_model_idx", "tenant_id", "provider_name", "model_type"),
     )
 
     id: Mapped[str] = mapped_column(StringUUID, server_default=text("uuid_generate_v4()"))
@@ -219,7 +219,7 @@ class LoadBalancingModelConfig(Base):
     model_name: Mapped[str] = mapped_column(String(255), nullable=False)
     model_type: Mapped[str] = mapped_column(String(40), nullable=False)
     name: Mapped[str] = mapped_column(String(255), nullable=False)
-    encrypted_config: Mapped[Optional[str]] = mapped_column(db.Text, nullable=True)
-    enabled: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=text("true"))
+    encrypted_config: Mapped[Optional[str]] = mapped_column(sa.Text, nullable=True)
+    enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=text("true"))
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())

api/models/source.py

@@ -2,50 +2,50 @@ import json
 from datetime import datetime
 from typing import Optional
 
+import sqlalchemy as sa
 from sqlalchemy import DateTime, String, func
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Mapped, mapped_column
 
 from models.base import Base
 
-from .engine import db
 from .types import StringUUID
 
 
 class DataSourceOauthBinding(Base):
     __tablename__ = "data_source_oauth_bindings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="source_binding_pkey"),
-        db.Index("source_binding_tenant_id_idx", "tenant_id"),
-        db.Index("source_info_idx", "source_info", postgresql_using="gin"),
+        sa.PrimaryKeyConstraint("id", name="source_binding_pkey"),
+        sa.Index("source_binding_tenant_id_idx", "tenant_id"),
+        sa.Index("source_info_idx", "source_info", postgresql_using="gin"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     access_token: Mapped[str] = mapped_column(String(255), nullable=False)
     provider: Mapped[str] = mapped_column(String(255), nullable=False)
     source_info = mapped_column(JSONB, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
-    disabled: Mapped[Optional[bool]] = mapped_column(db.Boolean, nullable=True, server_default=db.text("false"))
+    disabled: Mapped[Optional[bool]] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"))
 
 
 class DataSourceApiKeyAuthBinding(Base):
     __tablename__ = "data_source_api_key_auth_bindings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="data_source_api_key_auth_binding_pkey"),
-        db.Index("data_source_api_key_auth_binding_tenant_id_idx", "tenant_id"),
-        db.Index("data_source_api_key_auth_binding_provider_idx", "provider"),
+        sa.PrimaryKeyConstraint("id", name="data_source_api_key_auth_binding_pkey"),
+        sa.Index("data_source_api_key_auth_binding_tenant_id_idx", "tenant_id"),
+        sa.Index("data_source_api_key_auth_binding_provider_idx", "provider"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id = mapped_column(StringUUID, nullable=False)
     category: Mapped[str] = mapped_column(String(255), nullable=False)
     provider: Mapped[str] = mapped_column(String(255), nullable=False)
-    credentials = mapped_column(db.Text, nullable=True)  # JSON
+    credentials = mapped_column(sa.Text, nullable=True)  # JSON
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
-    disabled: Mapped[Optional[bool]] = mapped_column(db.Boolean, nullable=True, server_default=db.text("false"))
+    disabled: Mapped[Optional[bool]] = mapped_column(sa.Boolean, nullable=True, server_default=sa.text("false"))
 
     def to_dict(self):
         return {

api/models/task.py

@@ -1,6 +1,7 @@
 from datetime import datetime
 from typing import Optional
 
+import sqlalchemy as sa
 from celery import states  # type: ignore
 from sqlalchemy import DateTime, String
 from sqlalchemy.orm import Mapped, mapped_column
@@ -16,7 +17,7 @@ class CeleryTask(Base):
 
     __tablename__ = "celery_taskmeta"
 
-    id = mapped_column(db.Integer, db.Sequence("task_id_sequence"), primary_key=True, autoincrement=True)
+    id = mapped_column(sa.Integer, sa.Sequence("task_id_sequence"), primary_key=True, autoincrement=True)
     task_id = mapped_column(String(155), unique=True)
     status = mapped_column(String(50), default=states.PENDING)
     result = mapped_column(db.PickleType, nullable=True)
@@ -26,12 +27,12 @@ class CeleryTask(Base):
         onupdate=lambda: naive_utc_now(),
         nullable=True,
     )
-    traceback = mapped_column(db.Text, nullable=True)
+    traceback = mapped_column(sa.Text, nullable=True)
     name = mapped_column(String(155), nullable=True)
-    args = mapped_column(db.LargeBinary, nullable=True)
-    kwargs = mapped_column(db.LargeBinary, nullable=True)
+    args = mapped_column(sa.LargeBinary, nullable=True)
+    kwargs = mapped_column(sa.LargeBinary, nullable=True)
     worker = mapped_column(String(155), nullable=True)
-    retries: Mapped[Optional[int]] = mapped_column(db.Integer, nullable=True)
+    retries: Mapped[Optional[int]] = mapped_column(sa.Integer, nullable=True)
     queue = mapped_column(String(155), nullable=True)
 
 
@@ -41,7 +42,7 @@ class CeleryTaskSet(Base):
     __tablename__ = "celery_tasksetmeta"
 
     id: Mapped[int] = mapped_column(
-        db.Integer, db.Sequence("taskset_id_sequence"), autoincrement=True, primary_key=True
+        sa.Integer, sa.Sequence("taskset_id_sequence"), autoincrement=True, primary_key=True
     )
     taskset_id = mapped_column(String(155), unique=True)
     result = mapped_column(db.PickleType, nullable=True)

api/models/tools.py

@@ -25,33 +25,33 @@ from .types import StringUUID
 class ToolOAuthSystemClient(Base):
     __tablename__ = "tool_oauth_system_clients"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tool_oauth_system_client_pkey"),
-        db.UniqueConstraint("plugin_id", "provider", name="tool_oauth_system_client_plugin_id_provider_idx"),
+        sa.PrimaryKeyConstraint("id", name="tool_oauth_system_client_pkey"),
+        sa.UniqueConstraint("plugin_id", "provider", name="tool_oauth_system_client_plugin_id_provider_idx"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     plugin_id = mapped_column(String(512), nullable=False)
     provider: Mapped[str] = mapped_column(String(255), nullable=False)
     # oauth params of the tool provider
-    encrypted_oauth_params: Mapped[str] = mapped_column(db.Text, nullable=False)
+    encrypted_oauth_params: Mapped[str] = mapped_column(sa.Text, nullable=False)
 
 
 # tenant level tool oauth client params (client_id, client_secret, etc.)
 class ToolOAuthTenantClient(Base):
     __tablename__ = "tool_oauth_tenant_clients"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tool_oauth_tenant_client_pkey"),
-        db.UniqueConstraint("tenant_id", "plugin_id", "provider", name="unique_tool_oauth_tenant_client"),
+        sa.PrimaryKeyConstraint("id", name="tool_oauth_tenant_client_pkey"),
+        sa.UniqueConstraint("tenant_id", "plugin_id", "provider", name="unique_tool_oauth_tenant_client"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     # tenant id
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     plugin_id: Mapped[str] = mapped_column(String(512), nullable=False)
     provider: Mapped[str] = mapped_column(String(255), nullable=False)
-    enabled: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("true"))
+    enabled: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("true"))
     # oauth params of the tool provider
-    encrypted_oauth_params: Mapped[str] = mapped_column(db.Text, nullable=False)
+    encrypted_oauth_params: Mapped[str] = mapped_column(sa.Text, nullable=False)
 
     @property
     def oauth_params(self) -> dict:
@@ -65,14 +65,14 @@ class BuiltinToolProvider(Base):
 
     __tablename__ = "tool_builtin_providers"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tool_builtin_provider_pkey"),
-        db.UniqueConstraint("tenant_id", "provider", "name", name="unique_builtin_tool_provider"),
+        sa.PrimaryKeyConstraint("id", name="tool_builtin_provider_pkey"),
+        sa.UniqueConstraint("tenant_id", "provider", "name", name="unique_builtin_tool_provider"),
     )
 
     # id of the tool provider
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     name: Mapped[str] = mapped_column(
-        String(256), nullable=False, server_default=db.text("'API KEY 1'::character varying")
+        String(256), nullable=False, server_default=sa.text("'API KEY 1'::character varying")
     )
     # id of the tenant
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=True)
@@ -81,19 +81,19 @@ class BuiltinToolProvider(Base):
     # name of the tool provider
     provider: Mapped[str] = mapped_column(String(256), nullable=False)
     # credential of the tool provider
-    encrypted_credentials: Mapped[str] = mapped_column(db.Text, nullable=True)
+    encrypted_credentials: Mapped[str] = mapped_column(sa.Text, nullable=True)
     created_at: Mapped[datetime] = mapped_column(
-        sa.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
     updated_at: Mapped[datetime] = mapped_column(
-        sa.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
-    is_default: Mapped[bool] = mapped_column(db.Boolean, nullable=False, server_default=db.text("false"))
+    is_default: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, server_default=sa.text("false"))
     # credential type, e.g., "api-key", "oauth2"
     credential_type: Mapped[str] = mapped_column(
-        String(32), nullable=False, server_default=db.text("'api-key'::character varying")
+        String(32), nullable=False, server_default=sa.text("'api-key'::character varying")
     )
-    expires_at: Mapped[int] = mapped_column(db.BigInteger, nullable=False, server_default=db.text("-1"))
+    expires_at: Mapped[int] = mapped_column(sa.BigInteger, nullable=False, server_default=sa.text("-1"))
 
     @property
     def credentials(self) -> dict:
@@ -107,28 +107,28 @@ class ApiToolProvider(Base):
 
     __tablename__ = "tool_api_providers"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tool_api_provider_pkey"),
-        db.UniqueConstraint("name", "tenant_id", name="unique_api_tool_provider"),
+        sa.PrimaryKeyConstraint("id", name="tool_api_provider_pkey"),
+        sa.UniqueConstraint("name", "tenant_id", name="unique_api_tool_provider"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     # name of the api provider
-    name = mapped_column(String(255), nullable=False, server_default=db.text("'API KEY 1'::character varying"))
+    name = mapped_column(String(255), nullable=False, server_default=sa.text("'API KEY 1'::character varying"))
     # icon
     icon: Mapped[str] = mapped_column(String(255), nullable=False)
     # original schema
-    schema = mapped_column(db.Text, nullable=False)
+    schema = mapped_column(sa.Text, nullable=False)
     schema_type_str: Mapped[str] = mapped_column(String(40), nullable=False)
     # who created this tool
     user_id = mapped_column(StringUUID, nullable=False)
     # tenant id
     tenant_id = mapped_column(StringUUID, nullable=False)
     # description of the provider
-    description = mapped_column(db.Text, nullable=False)
+    description = mapped_column(sa.Text, nullable=False)
     # json format tools
-    tools_str = mapped_column(db.Text, nullable=False)
+    tools_str = mapped_column(sa.Text, nullable=False)
     # json format credentials
-    credentials_str = mapped_column(db.Text, nullable=False)
+    credentials_str = mapped_column(sa.Text, nullable=False)
     # privacy policy
     privacy_policy = mapped_column(String(255), nullable=True)
     # custom_disclaimer
@@ -167,11 +167,11 @@ class ToolLabelBinding(Base):
 
     __tablename__ = "tool_label_bindings"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tool_label_bind_pkey"),
-        db.UniqueConstraint("tool_id", "label_name", name="unique_tool_label_bind"),
+        sa.PrimaryKeyConstraint("id", name="tool_label_bind_pkey"),
+        sa.UniqueConstraint("tool_id", "label_name", name="unique_tool_label_bind"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     # tool id
     tool_id: Mapped[str] = mapped_column(String(64), nullable=False)
     # tool type
@@ -187,12 +187,12 @@ class WorkflowToolProvider(Base):
 
     __tablename__ = "tool_workflow_providers"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tool_workflow_provider_pkey"),
-        db.UniqueConstraint("name", "tenant_id", name="unique_workflow_tool_provider"),
-        db.UniqueConstraint("tenant_id", "app_id", name="unique_workflow_tool_provider_app_id"),
+        sa.PrimaryKeyConstraint("id", name="tool_workflow_provider_pkey"),
+        sa.UniqueConstraint("name", "tenant_id", name="unique_workflow_tool_provider"),
+        sa.UniqueConstraint("tenant_id", "app_id", name="unique_workflow_tool_provider_app_id"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     # name of the workflow provider
     name: Mapped[str] = mapped_column(String(255), nullable=False)
     # label of the workflow provider
@@ -208,17 +208,17 @@ class WorkflowToolProvider(Base):
     # tenant id
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     # description of the provider
-    description: Mapped[str] = mapped_column(db.Text, nullable=False)
+    description: Mapped[str] = mapped_column(sa.Text, nullable=False)
     # parameter configuration
-    parameter_configuration: Mapped[str] = mapped_column(db.Text, nullable=False, server_default="[]")
+    parameter_configuration: Mapped[str] = mapped_column(sa.Text, nullable=False, server_default="[]")
     # privacy policy
     privacy_policy: Mapped[str] = mapped_column(String(255), nullable=True, server_default="")
 
     created_at: Mapped[datetime] = mapped_column(
-        sa.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
     updated_at: Mapped[datetime] = mapped_column(
-        sa.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
 
     @property
@@ -245,19 +245,19 @@ class MCPToolProvider(Base):
 
     __tablename__ = "tool_mcp_providers"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tool_mcp_provider_pkey"),
-        db.UniqueConstraint("tenant_id", "server_url_hash", name="unique_mcp_provider_server_url"),
-        db.UniqueConstraint("tenant_id", "name", name="unique_mcp_provider_name"),
-        db.UniqueConstraint("tenant_id", "server_identifier", name="unique_mcp_provider_server_identifier"),
+        sa.PrimaryKeyConstraint("id", name="tool_mcp_provider_pkey"),
+        sa.UniqueConstraint("tenant_id", "server_url_hash", name="unique_mcp_provider_server_url"),
+        sa.UniqueConstraint("tenant_id", "name", name="unique_mcp_provider_name"),
+        sa.UniqueConstraint("tenant_id", "server_identifier", name="unique_mcp_provider_server_identifier"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     # name of the mcp provider
     name: Mapped[str] = mapped_column(String(40), nullable=False)
     # server identifier of the mcp provider
     server_identifier: Mapped[str] = mapped_column(String(64), nullable=False)
     # encrypted url of the mcp provider
-    server_url: Mapped[str] = mapped_column(db.Text, nullable=False)
+    server_url: Mapped[str] = mapped_column(sa.Text, nullable=False)
     # hash of server_url for uniqueness check
     server_url_hash: Mapped[str] = mapped_column(String(64), nullable=False)
     # icon of the mcp provider
@@ -267,16 +267,16 @@ class MCPToolProvider(Base):
     # who created this tool
     user_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     # encrypted credentials
-    encrypted_credentials: Mapped[str] = mapped_column(db.Text, nullable=True)
+    encrypted_credentials: Mapped[str] = mapped_column(sa.Text, nullable=True)
     # authed
-    authed: Mapped[bool] = mapped_column(db.Boolean, nullable=False, default=False)
+    authed: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=False)
     # tools
-    tools: Mapped[str] = mapped_column(db.Text, nullable=False, default="[]")
+    tools: Mapped[str] = mapped_column(sa.Text, nullable=False, default="[]")
     created_at: Mapped[datetime] = mapped_column(
-        sa.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
     updated_at: Mapped[datetime] = mapped_column(
-        sa.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)")
+        sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)")
     )
 
     def load_user(self) -> Account | None:
@@ -347,9 +347,9 @@ class ToolModelInvoke(Base):
     """
 
     __tablename__ = "tool_model_invokes"
-    __table_args__ = (db.PrimaryKeyConstraint("id", name="tool_model_invoke_pkey"),)
+    __table_args__ = (sa.PrimaryKeyConstraint("id", name="tool_model_invoke_pkey"),)
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     # who invoke this tool
     user_id = mapped_column(StringUUID, nullable=False)
     # tenant id
@@ -361,18 +361,18 @@ class ToolModelInvoke(Base):
     # tool name
     tool_name = mapped_column(String(128), nullable=False)
     # invoke parameters
-    model_parameters = mapped_column(db.Text, nullable=False)
+    model_parameters = mapped_column(sa.Text, nullable=False)
     # prompt messages
-    prompt_messages = mapped_column(db.Text, nullable=False)
+    prompt_messages = mapped_column(sa.Text, nullable=False)
     # invoke response
-    model_response = mapped_column(db.Text, nullable=False)
+    model_response = mapped_column(sa.Text, nullable=False)
 
-    prompt_tokens: Mapped[int] = mapped_column(db.Integer, nullable=False, server_default=db.text("0"))
-    answer_tokens: Mapped[int] = mapped_column(db.Integer, nullable=False, server_default=db.text("0"))
-    answer_unit_price = mapped_column(db.Numeric(10, 4), nullable=False)
-    answer_price_unit = mapped_column(db.Numeric(10, 7), nullable=False, server_default=db.text("0.001"))
-    provider_response_latency = mapped_column(db.Float, nullable=False, server_default=db.text("0"))
-    total_price = mapped_column(db.Numeric(10, 7))
+    prompt_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"))
+    answer_tokens: Mapped[int] = mapped_column(sa.Integer, nullable=False, server_default=sa.text("0"))
+    answer_unit_price = mapped_column(sa.Numeric(10, 4), nullable=False)
+    answer_price_unit = mapped_column(sa.Numeric(10, 7), nullable=False, server_default=sa.text("0.001"))
+    provider_response_latency = mapped_column(sa.Float, nullable=False, server_default=sa.text("0"))
+    total_price = mapped_column(sa.Numeric(10, 7))
     currency: Mapped[str] = mapped_column(String(255), nullable=False)
     created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
@@ -386,13 +386,13 @@ class ToolConversationVariables(Base):
 
     __tablename__ = "tool_conversation_variables"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tool_conversation_variables_pkey"),
+        sa.PrimaryKeyConstraint("id", name="tool_conversation_variables_pkey"),
         # add index for user_id and conversation_id
-        db.Index("user_id_idx", "user_id"),
-        db.Index("conversation_id_idx", "conversation_id"),
+        sa.Index("user_id_idx", "user_id"),
+        sa.Index("conversation_id_idx", "conversation_id"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     # conversation user id
     user_id = mapped_column(StringUUID, nullable=False)
     # tenant id
@@ -400,7 +400,7 @@ class ToolConversationVariables(Base):
     # conversation id
     conversation_id = mapped_column(StringUUID, nullable=False)
     # variables pool
-    variables_str = mapped_column(db.Text, nullable=False)
+    variables_str = mapped_column(sa.Text, nullable=False)
 
     created_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
     updated_at = mapped_column(sa.DateTime, nullable=False, server_default=func.current_timestamp())
@@ -417,11 +417,11 @@ class ToolFile(Base):
 
     __tablename__ = "tool_files"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="tool_file_pkey"),
-        db.Index("tool_file_conversation_id_idx", "conversation_id"),
+        sa.PrimaryKeyConstraint("id", name="tool_file_pkey"),
+        sa.Index("tool_file_conversation_id_idx", "conversation_id"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     # conversation user id
     user_id: Mapped[str] = mapped_column(StringUUID)
     # tenant id
@@ -448,30 +448,30 @@ class DeprecatedPublishedAppTool(Base):
 
     __tablename__ = "tool_published_apps"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="published_app_tool_pkey"),
-        db.UniqueConstraint("app_id", "user_id", name="unique_published_app_tool"),
+        sa.PrimaryKeyConstraint("id", name="published_app_tool_pkey"),
+        sa.UniqueConstraint("app_id", "user_id", name="unique_published_app_tool"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     # id of the app
     app_id = mapped_column(StringUUID, ForeignKey("apps.id"), nullable=False)
 
     user_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     # who published this tool
-    description = mapped_column(db.Text, nullable=False)
+    description = mapped_column(sa.Text, nullable=False)
     # llm_description of the tool, for LLM
-    llm_description = mapped_column(db.Text, nullable=False)
+    llm_description = mapped_column(sa.Text, nullable=False)
     # query description, query will be seem as a parameter of the tool,
     # to describe this parameter to llm, we need this field
-    query_description = mapped_column(db.Text, nullable=False)
+    query_description = mapped_column(sa.Text, nullable=False)
     # query name, the name of the query parameter
     query_name = mapped_column(String(40), nullable=False)
     # name of the tool provider
     tool_name = mapped_column(String(40), nullable=False)
     # author
     author = mapped_column(String(40), nullable=False)
-    created_at = mapped_column(sa.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))
-    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=db.text("CURRENT_TIMESTAMP(0)"))
+    created_at = mapped_column(sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"))
+    updated_at = mapped_column(sa.DateTime, nullable=False, server_default=sa.text("CURRENT_TIMESTAMP(0)"))
 
     @property
     def description_i18n(self) -> I18nObject:

api/models/web.py

@@ -1,5 +1,6 @@
 from datetime import datetime
 
+import sqlalchemy as sa
 from sqlalchemy import DateTime, String, func
 from sqlalchemy.orm import Mapped, mapped_column
 
@@ -13,15 +14,15 @@ from .types import StringUUID
 class SavedMessage(Base):
     __tablename__ = "saved_messages"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="saved_message_pkey"),
-        db.Index("saved_message_message_idx", "app_id", "message_id", "created_by_role", "created_by"),
+        sa.PrimaryKeyConstraint("id", name="saved_message_pkey"),
+        sa.Index("saved_message_message_idx", "app_id", "message_id", "created_by_role", "created_by"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     message_id = mapped_column(StringUUID, nullable=False)
     created_by_role = mapped_column(
-        String(255), nullable=False, server_default=db.text("'end_user'::character varying")
+        String(255), nullable=False, server_default=sa.text("'end_user'::character varying")
     )
     created_by = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
@@ -34,15 +35,15 @@ class SavedMessage(Base):
 class PinnedConversation(Base):
     __tablename__ = "pinned_conversations"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="pinned_conversation_pkey"),
-        db.Index("pinned_conversation_conversation_idx", "app_id", "conversation_id", "created_by_role", "created_by"),
+        sa.PrimaryKeyConstraint("id", name="pinned_conversation_pkey"),
+        sa.Index("pinned_conversation_conversation_idx", "app_id", "conversation_id", "created_by_role", "created_by"),
     )
 
-    id = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     app_id = mapped_column(StringUUID, nullable=False)
     conversation_id: Mapped[str] = mapped_column(StringUUID)
     created_by_role = mapped_column(
-        String(255), nullable=False, server_default=db.text("'end_user'::character varying")
+        String(255), nullable=False, server_default=sa.text("'end_user'::character varying")
     )
     created_by = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())

api/models/workflow.py

@@ -6,6 +6,7 @@ from enum import Enum, StrEnum
 from typing import TYPE_CHECKING, Any, Optional, Union
 from uuid import uuid4
 
+import sqlalchemy as sa
 from flask_login import current_user
 from sqlalchemy import DateTime, orm
 
@@ -24,7 +25,6 @@ from ._workflow_exc import NodeNotFoundError, WorkflowDataError
 if TYPE_CHECKING:
     from models.model import AppMode
 
-import sqlalchemy as sa
 from sqlalchemy import Index, PrimaryKeyConstraint, String, UniqueConstraint, func
 from sqlalchemy.orm import Mapped, declared_attr, mapped_column
 
@@ -117,11 +117,11 @@ class Workflow(Base):
 
     __tablename__ = "workflows"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="workflow_pkey"),
-        db.Index("workflow_version_idx", "tenant_id", "app_id", "version"),
+        sa.PrimaryKeyConstraint("id", name="workflow_pkey"),
+        sa.Index("workflow_version_idx", "tenant_id", "app_id", "version"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
     type: Mapped[str] = mapped_column(String(255), nullable=False)
@@ -140,10 +140,10 @@ class Workflow(Base):
         server_onupdate=func.current_timestamp(),
     )
     _environment_variables: Mapped[str] = mapped_column(
-        "environment_variables", db.Text, nullable=False, server_default="{}"
+        "environment_variables", sa.Text, nullable=False, server_default="{}"
     )
     _conversation_variables: Mapped[str] = mapped_column(
-        "conversation_variables", db.Text, nullable=False, server_default="{}"
+        "conversation_variables", sa.Text, nullable=False, server_default="{}"
     )
 
     VERSION_DRAFT = "draft"
@@ -491,11 +491,11 @@ class WorkflowRun(Base):
 
     __tablename__ = "workflow_runs"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="workflow_run_pkey"),
-        db.Index("workflow_run_triggerd_from_idx", "tenant_id", "app_id", "triggered_from"),
+        sa.PrimaryKeyConstraint("id", name="workflow_run_pkey"),
+        sa.Index("workflow_run_triggerd_from_idx", "tenant_id", "app_id", "triggered_from"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID)
     app_id: Mapped[str] = mapped_column(StringUUID)
 
@@ -503,19 +503,19 @@ class WorkflowRun(Base):
     type: Mapped[str] = mapped_column(String(255))
     triggered_from: Mapped[str] = mapped_column(String(255))
     version: Mapped[str] = mapped_column(String(255))
-    graph: Mapped[Optional[str]] = mapped_column(db.Text)
-    inputs: Mapped[Optional[str]] = mapped_column(db.Text)
+    graph: Mapped[Optional[str]] = mapped_column(sa.Text)
+    inputs: Mapped[Optional[str]] = mapped_column(sa.Text)
     status: Mapped[str] = mapped_column(String(255))  # running, succeeded, failed, stopped, partial-succeeded
     outputs: Mapped[Optional[str]] = mapped_column(sa.Text, default="{}")
-    error: Mapped[Optional[str]] = mapped_column(db.Text)
-    elapsed_time: Mapped[float] = mapped_column(db.Float, nullable=False, server_default=sa.text("0"))
+    error: Mapped[Optional[str]] = mapped_column(sa.Text)
+    elapsed_time: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("0"))
     total_tokens: Mapped[int] = mapped_column(sa.BigInteger, server_default=sa.text("0"))
-    total_steps: Mapped[int] = mapped_column(db.Integer, server_default=db.text("0"), nullable=True)
+    total_steps: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"), nullable=True)
     created_by_role: Mapped[str] = mapped_column(String(255))  # account, end_user
     created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
     created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
     finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
-    exceptions_count: Mapped[int] = mapped_column(db.Integer, server_default=db.text("0"), nullable=True)
+    exceptions_count: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"), nullable=True)
 
     @property
     def created_by_account(self):
@@ -704,25 +704,25 @@ class WorkflowNodeExecutionModel(Base):
             ),
         )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID)
     app_id: Mapped[str] = mapped_column(StringUUID)
     workflow_id: Mapped[str] = mapped_column(StringUUID)
     triggered_from: Mapped[str] = mapped_column(String(255))
     workflow_run_id: Mapped[Optional[str]] = mapped_column(StringUUID)
-    index: Mapped[int] = mapped_column(db.Integer)
+    index: Mapped[int] = mapped_column(sa.Integer)
     predecessor_node_id: Mapped[Optional[str]] = mapped_column(String(255))
     node_execution_id: Mapped[Optional[str]] = mapped_column(String(255))
     node_id: Mapped[str] = mapped_column(String(255))
     node_type: Mapped[str] = mapped_column(String(255))
     title: Mapped[str] = mapped_column(String(255))
-    inputs: Mapped[Optional[str]] = mapped_column(db.Text)
-    process_data: Mapped[Optional[str]] = mapped_column(db.Text)
-    outputs: Mapped[Optional[str]] = mapped_column(db.Text)
+    inputs: Mapped[Optional[str]] = mapped_column(sa.Text)
+    process_data: Mapped[Optional[str]] = mapped_column(sa.Text)
+    outputs: Mapped[Optional[str]] = mapped_column(sa.Text)
     status: Mapped[str] = mapped_column(String(255))
-    error: Mapped[Optional[str]] = mapped_column(db.Text)
-    elapsed_time: Mapped[float] = mapped_column(db.Float, server_default=db.text("0"))
-    execution_metadata: Mapped[Optional[str]] = mapped_column(db.Text)
+    error: Mapped[Optional[str]] = mapped_column(sa.Text)
+    elapsed_time: Mapped[float] = mapped_column(sa.Float, server_default=sa.text("0"))
+    execution_metadata: Mapped[Optional[str]] = mapped_column(sa.Text)
     created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp())
     created_by_role: Mapped[str] = mapped_column(String(255))
     created_by: Mapped[str] = mapped_column(StringUUID)
@@ -834,11 +834,11 @@ class WorkflowAppLog(Base):
 
     __tablename__ = "workflow_app_logs"
     __table_args__ = (
-        db.PrimaryKeyConstraint("id", name="workflow_app_log_pkey"),
-        db.Index("workflow_app_log_app_idx", "tenant_id", "app_id"),
+        sa.PrimaryKeyConstraint("id", name="workflow_app_log_pkey"),
+        sa.Index("workflow_app_log_app_idx", "tenant_id", "app_id"),
     )
 
-    id: Mapped[str] = mapped_column(StringUUID, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
     tenant_id: Mapped[str] = mapped_column(StringUUID)
     app_id: Mapped[str] = mapped_column(StringUUID)
     workflow_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
@@ -864,6 +864,19 @@ class WorkflowAppLog(Base):
         created_by_role = CreatorUserRole(self.created_by_role)
         return db.session.get(EndUser, self.created_by) if created_by_role == CreatorUserRole.END_USER else None
 
+    def to_dict(self):
+        return {
+            "id": self.id,
+            "tenant_id": self.tenant_id,
+            "app_id": self.app_id,
+            "workflow_id": self.workflow_id,
+            "workflow_run_id": self.workflow_run_id,
+            "created_from": self.created_from,
+            "created_by_role": self.created_by_role,
+            "created_by": self.created_by,
+            "created_at": self.created_at,
+        }
+
 
 class ConversationVariable(Base):
     __tablename__ = "workflow_conversation_variables"
@@ -871,7 +884,7 @@ class ConversationVariable(Base):
     id: Mapped[str] = mapped_column(StringUUID, primary_key=True)
     conversation_id: Mapped[str] = mapped_column(StringUUID, nullable=False, primary_key=True, index=True)
     app_id: Mapped[str] = mapped_column(StringUUID, nullable=False, index=True)
-    data: Mapped[str] = mapped_column(db.Text, nullable=False)
+    data: Mapped[str] = mapped_column(sa.Text, nullable=False)
     created_at: Mapped[datetime] = mapped_column(
         DateTime, nullable=False, server_default=func.current_timestamp(), index=True
     )
@@ -933,7 +946,7 @@ class WorkflowDraftVariable(Base):
     __allow_unmapped__ = True
 
     # id is the unique identifier of a draft variable.
-    id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=db.text("uuid_generate_v4()"))
+    id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
 
     created_at: Mapped[datetime] = mapped_column(
         DateTime,

api/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dify-api"
-version = "1.7.1"
+version = "1.7.2"
 requires-python = ">=3.11,<3.13"
 
 dependencies = [
@@ -49,6 +49,8 @@ dependencies = [
     "opentelemetry-instrumentation==0.48b0",
     "opentelemetry-instrumentation-celery==0.48b0",
     "opentelemetry-instrumentation-flask==0.48b0",
+    "opentelemetry-instrumentation-redis==0.48b0",
+    "opentelemetry-instrumentation-requests==0.48b0",
     "opentelemetry-instrumentation-sqlalchemy==0.48b0",
     "opentelemetry-propagator-b3==1.27.0",
     # opentelemetry-proto1.28.0 depends on protobuf (>=5.0,<6.0),
@@ -114,6 +116,7 @@ dev = [
     "pytest-cov~=4.1.0",
     "pytest-env~=1.1.3",
     "pytest-mock~=3.14.0",
+    "testcontainers~=4.10.0",
     "types-aiofiles~=24.1.0",
     "types-beautifulsoup4~=4.12.0",
     "types-cachetools~=5.5.0",
@@ -191,6 +194,7 @@ vdb = [
     "alibabacloud_tea_openapi~=0.3.9",
     "chromadb==0.5.20",
     "clickhouse-connect~=0.7.16",
+    "clickzetta-connector-python>=0.8.102",
     "couchbase~=4.3.0",
     "elasticsearch==8.14.0",
     "opensearch-py==2.4.0",
@@ -210,3 +214,4 @@ vdb = [
     "xinference-client~=1.2.2",
     "mo-vector~=0.1.13",
 ]
+

api/repositories/factory.py

@@ -48,7 +48,6 @@ class DifyAPIRepositoryFactory(DifyCoreRepositoryFactory):
             RepositoryImportError: If the configured repository cannot be imported or instantiated
         """
         class_path = dify_config.API_WORKFLOW_NODE_EXECUTION_REPOSITORY
-        logger.debug("Creating DifyAPIWorkflowNodeExecutionRepository from: %s", class_path)
 
         try:
             repository_class = cls._import_class(class_path)
@@ -86,7 +85,6 @@ class DifyAPIRepositoryFactory(DifyCoreRepositoryFactory):
             RepositoryImportError: If the configured repository cannot be imported or instantiated
         """
         class_path = dify_config.API_WORKFLOW_RUN_REPOSITORY
-        logger.debug("Creating APIWorkflowRunRepository from: %s", class_path)
 
         try:
             repository_class = cls._import_class(class_path)

api/schedule/clean_embedding_cache_task.py

@@ -3,7 +3,7 @@ import time
 
 import click
 from sqlalchemy import text
-from werkzeug.exceptions import NotFound
+from sqlalchemy.exc import SQLAlchemyError
 
 import app
 from configs import dify_config
@@ -27,8 +27,8 @@ def clean_embedding_cache_task():
                 .all()
             )
             embedding_ids = [embedding_id[0] for embedding_id in embedding_ids]
-        except NotFound:
-            break
+        except SQLAlchemyError:
+            raise
         if embedding_ids:
             for embedding_id in embedding_ids:
                 db.session.execute(

api/schedule/clean_messages.py

@@ -3,7 +3,7 @@ import logging
 import time
 
 import click
-from werkzeug.exceptions import NotFound
+from sqlalchemy.exc import SQLAlchemyError
 
 import app
 from configs import dify_config
@@ -42,8 +42,8 @@ def clean_messages():
                 .all()
             )
 
-        except NotFound:
-            break
+        except SQLAlchemyError:
+            raise
         if not messages:
             break
         for message in messages:

api/schedule/clean_unused_datasets_task.py

@@ -3,7 +3,7 @@ import time
 
 import click
 from sqlalchemy import func, select
-from werkzeug.exceptions import NotFound
+from sqlalchemy.exc import SQLAlchemyError
 
 import app
 from configs import dify_config
@@ -65,8 +65,8 @@ def clean_unused_datasets_task():
 
             datasets = db.paginate(stmt, page=1, per_page=50)
 
-        except NotFound:
-            break
+        except SQLAlchemyError:
+            raise
         if datasets.items is None or len(datasets.items) == 0:
             break
         for dataset in datasets:
@@ -146,8 +146,8 @@ def clean_unused_datasets_task():
             )
             datasets = db.paginate(stmt, page=1, per_page=50)
 
-        except NotFound:
-            break
+        except SQLAlchemyError:
+            raise
         if datasets.items is None or len(datasets.items) == 0:
             break
         for dataset in datasets:

api/services/app_dsl_service.py

@@ -12,6 +12,7 @@ import yaml  # type: ignore
 from Crypto.Cipher import AES
 from Crypto.Util.Padding import pad, unpad
 from packaging import version
+from packaging.version import parse as parse_version
 from pydantic import BaseModel, Field
 from sqlalchemy import select
 from sqlalchemy.orm import Session
@@ -269,7 +270,7 @@ class AppDslService:
             check_dependencies_pending_data = None
             if dependencies:
                 check_dependencies_pending_data = [PluginDependency.model_validate(d) for d in dependencies]
-            elif imported_version <= "0.1.5":
+            elif parse_version(imported_version) <= parse_version("0.1.5"):
                 if "workflow" in data:
                     graph = data.get("workflow", {}).get("graph", {})
                     dependencies_list = self._extract_dependencies_from_workflow_graph(graph)

api/services/app_generate_service.py

@@ -1,5 +1,6 @@
+import uuid
 from collections.abc import Generator, Mapping
-from typing import Any, Union
+from typing import Any, Optional, Union
 
 from openai._exceptions import RateLimitError
 
@@ -15,6 +16,7 @@ from libs.helper import RateLimiter
 from models.model import Account, App, AppMode, EndUser
 from models.workflow import Workflow
 from services.billing_service import BillingService
+from services.errors.app import WorkflowIdFormatError, WorkflowNotFoundError
 from services.errors.llm import InvokeRateLimitError
 from services.workflow_service import WorkflowService
 
@@ -86,7 +88,8 @@ class AppGenerateService:
                     request_id=request_id,
                 )
             elif app_model.mode == AppMode.ADVANCED_CHAT.value:
-                workflow = cls._get_workflow(app_model, invoke_from)
+                workflow_id = args.get("workflow_id")
+                workflow = cls._get_workflow(app_model, invoke_from, workflow_id)
                 return rate_limit.generate(
                     AdvancedChatAppGenerator.convert_to_event_stream(
                         AdvancedChatAppGenerator().generate(
@@ -101,7 +104,8 @@ class AppGenerateService:
                     request_id=request_id,
                 )
             elif app_model.mode == AppMode.WORKFLOW.value:
-                workflow = cls._get_workflow(app_model, invoke_from)
+                workflow_id = args.get("workflow_id")
+                workflow = cls._get_workflow(app_model, invoke_from, workflow_id)
                 return rate_limit.generate(
                     WorkflowAppGenerator.convert_to_event_stream(
                         WorkflowAppGenerator().generate(
@@ -210,14 +214,27 @@ class AppGenerateService:
         )
 
     @classmethod
-    def _get_workflow(cls, app_model: App, invoke_from: InvokeFrom) -> Workflow:
+    def _get_workflow(cls, app_model: App, invoke_from: InvokeFrom, workflow_id: Optional[str] = None) -> Workflow:
         """
         Get workflow
         :param app_model: app model
         :param invoke_from: invoke from
+        :param workflow_id: optional workflow id to specify a specific version
         :return:
         """
         workflow_service = WorkflowService()
+
+        # If workflow_id is specified, get the specific workflow version
+        if workflow_id:
+            try:
+                workflow_uuid = uuid.UUID(workflow_id)
+            except ValueError:
+                raise WorkflowIdFormatError(f"Invalid workflow_id format: '{workflow_id}'. ")
+            workflow = workflow_service.get_published_workflow_by_id(app_model=app_model, workflow_id=workflow_id)
+            if not workflow:
+                raise WorkflowNotFoundError(f"Workflow not found with id: {workflow_id}")
+            return workflow
+
         if invoke_from == InvokeFrom.DEBUGGER:
             # fetch draft workflow by app_model
             workflow = workflow_service.get_draft_workflow(app_model=app_model)

api/services/billing_service.py

@@ -159,9 +159,9 @@ class BillingService:
     ):
         limiter_key = f"{account_id}:{tenant_id}"
         if cls.compliance_download_rate_limiter.is_rate_limited(limiter_key):
-            from controllers.console.error import CompilanceRateLimitError
+            from controllers.console.error import ComplianceRateLimitError
 
-            raise CompilanceRateLimitError()
+            raise ComplianceRateLimitError()
 
         json = {
             "doc_name": doc_name,

api/services/clear_free_plan_tenant_expired_logs.py

@@ -13,7 +13,19 @@ from core.model_runtime.utils.encoders import jsonable_encoder
 from extensions.ext_database import db
 from extensions.ext_storage import storage
 from models.account import Tenant
-from models.model import App, Conversation, Message
+from models.model import (
+    App,
+    AppAnnotationHitHistory,
+    Conversation,
+    Message,
+    MessageAgentThought,
+    MessageAnnotation,
+    MessageChain,
+    MessageFeedback,
+    MessageFile,
+)
+from models.web import SavedMessage
+from models.workflow import WorkflowAppLog
 from repositories.factory import DifyAPIRepositoryFactory
 from services.billing_service import BillingService
 
@@ -21,6 +33,85 @@ logger = logging.getLogger(__name__)
 
 
 class ClearFreePlanTenantExpiredLogs:
+    @classmethod
+    def _clear_message_related_tables(cls, session: Session, tenant_id: str, batch_message_ids: list[str]) -> None:
+        """
+        Clean up message-related tables to avoid data redundancy.
+        This method cleans up tables that have foreign key relationships with Message.
+
+        Args:
+            session: Database session, the same with the one in process_tenant method
+            tenant_id: Tenant ID for logging purposes
+            batch_message_ids: List of message IDs to clean up
+        """
+        if not batch_message_ids:
+            return
+
+        # Clean up each related table
+        related_tables = [
+            (MessageFeedback, "message_feedbacks"),
+            (MessageFile, "message_files"),
+            (MessageAnnotation, "message_annotations"),
+            (MessageChain, "message_chains"),
+            (MessageAgentThought, "message_agent_thoughts"),
+            (AppAnnotationHitHistory, "app_annotation_hit_histories"),
+            (SavedMessage, "saved_messages"),
+        ]
+
+        for model, table_name in related_tables:
+            # Query records related to expired messages
+            records = (
+                session.query(model)
+                .filter(
+                    model.message_id.in_(batch_message_ids),  # type: ignore
+                )
+                .all()
+            )
+
+            if len(records) == 0:
+                continue
+
+            # Save records before deletion
+            record_ids = [record.id for record in records]
+            try:
+                record_data = []
+                for record in records:
+                    try:
+                        if hasattr(record, "to_dict"):
+                            record_data.append(record.to_dict())
+                        else:
+                            # if record doesn't have to_dict method, we need to transform it to dict manually
+                            record_dict = {}
+                            for column in record.__table__.columns:
+                                record_dict[column.name] = getattr(record, column.name)
+                            record_data.append(record_dict)
+                    except Exception:
+                        logger.exception("Failed to transform %s record: %s", table_name, record.id)
+                        continue
+
+                if record_data:
+                    storage.save(
+                        f"free_plan_tenant_expired_logs/"
+                        f"{tenant_id}/{table_name}/{datetime.datetime.now().strftime('%Y-%m-%d')}"
+                        f"-{time.time()}.json",
+                        json.dumps(
+                            jsonable_encoder(record_data),
+                        ).encode("utf-8"),
+                    )
+            except Exception:
+                logger.exception("Failed to save %s records", table_name)
+
+            session.query(model).filter(
+                model.id.in_(record_ids),  # type: ignore
+            ).delete(synchronize_session=False)
+
+            click.echo(
+                click.style(
+                    f"[{datetime.datetime.now()}] Processed {len(record_ids)} "
+                    f"{table_name} records for tenant {tenant_id}"
+                )
+            )
+
     @classmethod
     def process_tenant(cls, flask_app: Flask, tenant_id: str, days: int, batch: int):
         with flask_app.app_context():
@@ -58,6 +149,7 @@ class ClearFreePlanTenantExpiredLogs:
                         Message.id.in_(message_ids),
                     ).delete(synchronize_session=False)
 
+                    cls._clear_message_related_tables(session, tenant_id, message_ids)
                     session.commit()
 
                     click.echo(
@@ -199,6 +291,48 @@ class ClearFreePlanTenantExpiredLogs:
                 if len(workflow_runs) < batch:
                     break
 
+            while True:
+                with Session(db.engine).no_autoflush as session:
+                    workflow_app_logs = (
+                        session.query(WorkflowAppLog)
+                        .filter(
+                            WorkflowAppLog.tenant_id == tenant_id,
+                            WorkflowAppLog.created_at < datetime.datetime.now() - datetime.timedelta(days=days),
+                        )
+                        .limit(batch)
+                        .all()
+                    )
+
+                    if len(workflow_app_logs) == 0:
+                        break
+
+                    # save workflow app logs
+                    storage.save(
+                        f"free_plan_tenant_expired_logs/"
+                        f"{tenant_id}/workflow_app_logs/{datetime.datetime.now().strftime('%Y-%m-%d')}"
+                        f"-{time.time()}.json",
+                        json.dumps(
+                            jsonable_encoder(
+                                [workflow_app_log.to_dict() for workflow_app_log in workflow_app_logs],
+                            ),
+                        ).encode("utf-8"),
+                    )
+
+                    workflow_app_log_ids = [workflow_app_log.id for workflow_app_log in workflow_app_logs]
+
+                    # delete workflow app logs
+                    session.query(WorkflowAppLog).filter(
+                        WorkflowAppLog.id.in_(workflow_app_log_ids),
+                    ).delete(synchronize_session=False)
+                    session.commit()
+
+                    click.echo(
+                        click.style(
+                            f"[{datetime.datetime.now()}] Processed {len(workflow_app_log_ids)}"
+                            f" workflow app logs for tenant {tenant_id}"
+                        )
+                    )
+
     @classmethod
     def process(cls, days: int, batch: int, tenant_ids: list[str]):
         """